Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4865 (2023-08-21 14:20:10)

add migration guide from Promscale
Follow up after: https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/commit/5aed369132b3e38d85bb29709b23d861503d9ff0

Signed-off-by: dmitryk-dk d.kozlov@victoriametrics.com


Patch:
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 775efe352..1eaadacfd 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -754,10 +754,10 @@ as soon as it is parsed in stream parsing mode.
 
 A single `vmagent` instance can scrape tens of thousands of scrape targets. Sometimes this isn't enough due to limitations on CPU, network, RAM, etc.
 In this case scrape targets can be split among multiple `vmagent` instances (aka `vmagent` horizontal scaling, sharding and clustering).
-Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values.
-The flag value must be in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster.
-The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag. For example, the following commands
-spread scrape targets among a cluster of two `vmagent` instances:
+The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag.
+Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values
+in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster specified via `-promscrape.cluster.membersCount`.
+For example, the following commands spread scrape targets among a cluster of two `vmagent` instances:
 
 ```
 /path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.config=/path/to/config.yml ...
@@ -765,7 +765,7 @@ spread scrape targets among a cluster of two `vmagent` instances:
 ```
 
 The `-promscrape.cluster.memberNum` can be set to a StatefulSet pod name when `vmagent` runs in Kubernetes.
-The pod name must end with a number in the range `0 ... promscrape.cluster.memberNum-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
+The pod name must end with a number in the range `0 ... promscrape.cluster.membersCount-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
 
 By default, each scrape target is scraped only by a single `vmagent` instance in the cluster. If there is a need for replicating scrape targets among multiple `vmagent` instances,
 then `-promscrape.cluster.replicationFactor` command-line flag must be set to the desired number of replicas. For example, the following commands
@@ -781,6 +781,14 @@ If each target is scraped by multiple `vmagent` instances, then data deduplicati
 The `-dedup.minScrapeInterval` must be set to the `scrape_interval` configured at `-promscrape.config`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
+The `-promscrape.cluster.memberLabel` command-line flag allows specifying a name for `member num` label to add to all the scraped metrics.
+The value of the `member num` label is set to `-promscrape.cluster.memberNum`. For example, the following config instructs adding `vmagent_instance="0"` label
+to all the metrics scraped by the given `vmagent` instance:
+
+```
+/path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.cluster.memberLabel=vmagent_instance
+```
+
 See also [how to shard data among multiple remote storage systems](#sharding-among-remote-storages).
 
 
@@ -1376,8 +1384,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
   -promscrape.azureSDCheckInterval duration
      Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#azure_sd_configs for details (default 1m0s)
+  -promscrape.cluster.memberLabel string
+     If non-empty, then the label with this name and the -promscrape.cluster.memberNum value is added to all the scraped metrics
   -promscrape.cluster.memberNum string
-     The number of number in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
+     The number of vmagent instance in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. See also -promscrape.cluster.memberLabel (default "0")
   -promscrape.cluster.membersCount int
      The number of members in a cluster of scrapers. Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets
   -promscrape.cluster.name string
diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go
index 05162e22d..1b5399dc0 100644
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -322,6 +323,19 @@ func (c *client) runWorker() {
 }
 
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+	req := c.newRequest(url, body)
+	resp, err := c.hc.Do(req)
+	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+		// it is likely connection became stale.
+		// So we do one more attempt in hope request will succeed.
+		// If not, the error should be handled by the caller as usual.
+		req = c.newRequest(url, body)
+		resp, err = c.hc.Do(req)
+	}
+	return resp, err
+}
+
+func (c *client) newRequest(url string, body []byte) *http.Request {
 	reqBody := bytes.NewBuffer(body)
 	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
@@ -345,7 +359,7 @@ func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
 			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
-	return c.hc.Do(req)
+	return req
 }
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go
index cf6a43d3e..6bac40be5 100644
--- a/app/vmalert/datasource/vm.go
+++ b/app/vmalert/datasource/vm.go
@@ -127,21 +127,14 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := s.newRequestPOST()
-	if err != nil {
-		return Result{}, nil, err
-	}
-
-	switch s.dataSourceType {
-	case "", datasourcePrometheus:
-		s.setPrometheusInstantReqParams(req, query, ts)
-	case datasourceGraphite:
-		s.setGraphiteReqParams(req, query, ts)
-	default:
-		return Result{}, nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
-	}
-
+	req := s.newQueryRequest(query, ts)
 	resp, err := s.do(ctx, req)
+	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+		// something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req = s.newQueryRequest(query, ts)
+		resp, err = s.do(ctx, req)
+	}
 	if err != nil {
 		return Result{}, req, err
 	}
@@ -164,18 +157,20 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if s.dataSourceType != datasourcePrometheus {
 		return res, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
 	}
-	req, err := s.newRequestPOST()
-	if err != nil {
-		return res, err
-	}
 	if start.IsZero() {
 		return res, fmt.Errorf("start param is missing")
 	}
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	s.setPrometheusRangeReqParams(req, query, start, end)
+	req := s.newQueryRangeRequest(query, start, end)
 	resp, err := s.do(ctx, req)
+	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+		// something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req = s.newQueryRangeRequest(query, start, end)
+		resp, err = s.do(ctx, req)
+	}
 	if err != nil {
 		return res, err
 	}
@@ -190,11 +185,6 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
 	}
 	resp, err := s.c.Do(req.WithContext(ctx))
-	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
-		// something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		resp, err = s.c.Do(req.WithContext(ctx))
-	}
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
@@ -206,10 +196,29 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	return resp, nil
 }
 
-func (s *VMStorage) newRequestPOST() (*http.Request, error) {
+func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) *http.Request {
+	req := s.newRequest()
+	s.setPrometheusRangeReqParams(req, query, start, end)
+	return req
+}
+
+func (s *VMStorage) newQueryRequest(query string, ts time.Time) *http.Request {
+	req := s.newRequest()
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
+		s.setPrometheusInstantReqParams(req, query, ts)
+	case datasourceGraphite:
+		s.setGraphiteReqParams(req, query, ts)
+	default:
+		logger.Panicf("BUG: engine not found: %q", s.dataSourceType)
+	}
+	return req
+}
+
+func (s *VMStorage) newRequest() *http.Request {
 	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
-		return nil, err
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
@@ -218,5 +227,5 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	for _, h := range s.extraHeaders {
 		req.Header.Set(h.key, h.value)
 	}
-	return req, nil
+	return req
 }
diff --git a/app/vmauth/README.md b/app/vmauth/README.md
index 48d0d8e35..7959e1c04 100644
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -221,6 +221,9 @@ ip_filters:
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
+Please note, vmauth doesn't follow redirects. If destination redirects request to a new location, make sure this 
+location is supported in vmauth `url_map` config.
+
 ## Security
 
 It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.
@@ -244,7 +247,7 @@ It is recommended protecting the following endpoints with authKeys:
 * `/metrics` with `metricsAuthKey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
 * `/debug/pprof` with `pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).
 
-`vmauth` also supports the ability to restict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
+`vmauth` also supports the ability to restrict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
 
 ## Monitoring
 
diff --git a/app/vmctl/README.md b/app/vmctl/README.md
index 50d11161e..9032dae1e 100644
--- a/app/vmctl/README.md
+++ b/app/vmctl/README.md
@@ -11,6 +11,7 @@ Features:
 - migrate data from [Mimir](#migrating-data-from-mimir) to VictoriaMetrics
 - migrate data from [InfluxDB](#migrating-data-from-influxdb-1x) to VictoriaMetrics
 - migrate data from [OpenTSDB](#migrating-data-from-opentsdb) to VictoriaMetrics
+- migrate data from [Promscale](#migrating-data-from-promscale)
 - migrate data between [VictoriaMetrics](#migrating-data-from-victoriametrics) single or cluster version.
 - migrate data by [Prometheus remote read protocol](#migrating-data-by-remote-read-protocol) to VictoriaMetrics
 - [verify](#verifying-exported-blocks-from-victoriametrics) exported blocks from VictoriaMetrics single or cluster version.
@@ -306,6 +307,41 @@ Please see more about time filtering [here](https://docs.influxdata.com/influxdb
 Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
 You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
 
+## Migrating data from Promscale
+
+[Promscale](https://github.com/timescale/promscale) supports [Prometheus Remote Read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+Hence, `vmctl` can be used in [remote-read](https://docs.victoriametrics.com/vmctl.html#migrating-data-by-remote-read-protocol)
+mode to migrate historical data from Promscale to VictoriaMetrics.
+
+See the example of migration command below:
+```console
+./vmctl remote-read --remote-read-src-addr=http://127.0.0.1:9201/read --remote-read-step-interval=day --remote-read-use-stream=false --vm-addr=http://127.0.0.1:8428 --remote-read-filter-time-start=2023-08-21T00:00:00Z --vm-concurrency=6 --remote-read-disable-path-append=true
+Selected time range "2023-08-21 00:00:00 +0000 UTC" - "2023-08-21 14:11:41.561979 +0000 UTC" will be split into 1 ranges according to "day" step. Continue? [Y/n] y
+VM worker 0:↙ 82831 samples/s                                                                                                                                                                        
+VM worker 1:↙ 54378 samples/s                                                                                                                                                                        
+VM worker 2:↙ 121616 samples/s                                                                                                                                                                       
+VM worker 3:↙ 59164 samples/s                                                                                                                                                                        
+VM worker 4:↙ 59220 samples/s                                                                                                                                                                        
+VM worker 5:↙ 102072 samples/s                                                                                                                                                                       
+Processing ranges: 1 / 1 [██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/08/21 16:11:55 Import finished!
+2023/08/21 16:11:55 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 14.047045459s;
+  total samples: 262111;
+  samples/s: 18659.51;
+  total bytes: 5.3 MB;
+  bytes/s: 376.4 kB;
+  import requests: 6;
+  import requests retries: 0;
+2023/08/21 16:11:55 Total time: 14.063458792s
+```
+
+Here we specify the full path to Promscale's Remote Read API via `--remote-read-src-addr` and we disable auto-path 
+appending via `--remote-read-disable-path-append` cmd-line flags. This is necessary, as Promscale has a different to 
+Prometheus API path. Promscale doesn't support stream mode, so it may consume additional memory during 
+the migration.
+
 ## Migrating data from Prometheus
 
 `vmctl` supports the `prometheus` mode for migrating data from Prometheus to VictoriaMetrics time-series database.
@@ -439,17 +475,27 @@ Found 2 blocks to import. Continue? [Y/n] y
 ## Migrating data by remote read protocol
 
 `vmctl` supports the `remote-read` mode for migrating data from databases which support 
-[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/)
+[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+
+Remote read API implemented two modes:
+1. `SAMPLES` - returns a message that includes a list of raw samples (one protocol buffer message);
+2. `STREAMED_XOR_CHUNKS` - sending a set of small protocol buffer messages;
+`STREAMED_XOR_CHUNKS` mode uses constant memory per request no matter how many samples was requested, when `SAMPLES` mode
+buffered the whole response of the remote read a raw, uncompressed format in order to marshsal it in a potentially 
+huge protobuf message before sending it to the client. So potentially `SAMPLES` mode may lead to allocate a lot of memory
+by both client and server each.
+For more information, please check this [blog](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/).  
 
 See `./vmctl remote-read --help` for details and full list of flags.
 
 To start the migration process configure the following flags:
 1. `--remote-read-src-addr` - data source address to read from;
-1. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
+2. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
    and for cluster version is equal to `--httpListenAddr` flag of vminsert component (for example `http://<vminsert>:8480/insert/<accountID>/prometheus`);
-1. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
-1. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
-1. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+3. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
+4. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
+5. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+6. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
 
 The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
@@ -516,7 +562,7 @@ and that you have a separate Thanos Store installation.
     - url: http://victoria-metrics:8428/api/v1/write
     ```
 
-1. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
+2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
     In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
 
     ```
@@ -524,7 +570,7 @@ and that you have a separate Thanos Store installation.
     2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2020_04" has been created
     ```
 
-1. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
+3. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
     storage.
 
 ### Historical data
@@ -533,12 +579,12 @@ Let's assume your data is stored on S3 served by minio. You first need to copy t
 then import it into VM using `vmctl` in `prometheus` mode.
 
 1. Copy data from minio.
-    1. Run the `minio/mc` Docker container.
-    1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
-    1. `mc cp -r minio/prometheus thanos-data`
-1. Import using `vmctl`.
-    1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
-    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
+    1.1 Run the `minio/mc` Docker container.
+    1.2 `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
+    1.3 `mc cp -r minio/prometheus thanos-data`
+2. Import using `vmctl`.
+    2.1 Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
+    2.2 Use [prometheus](#migrating-data-from-prometheus) mode to import data:
 
     ```
     vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
@@ -553,7 +599,7 @@ service (or anything that exposes gRPC StoreAPI e.g. Querier) via Prometheus rem
 If you want to migrate data, you should run [thanos-remote-read](https://github.com/G-Research/thanos-remote-read) proxy
 and define the Thanos store address `./thanos-remote-read -store 127.0.0.1:19194`. 
 It is important to know that `store` flag is Thanos Store API gRPC endpoint. 
-Also, it is important to know that thanos-remote-read proxy  doesn't support `STREAMED_XOR_CHUNKS` mode.
+Also, it is important to know that thanos-remote-read proxy doesn't support stream mode.
 When you run thanos-remote-read proxy, it exposes port to serve HTTP on `10080 by default`.
 
 The importing process example for local installation of Thanos
@@ -616,7 +662,7 @@ api:
 If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
 By default, Cortex uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
 
-It is important to know that Cortex doesn't support the `STREAMED_XOR_CHUNKS` mode.
+It is important to know that Cortex doesn't support the stream mode.
 When you run Cortex, it exposes a port to serve HTTP on `9009 by default`.
 
 The importing process example for the local installation of Cortex
@@ -674,7 +720,7 @@ api:
 If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
 By default, Mimir uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
 
-Mimir supports both remote read mode, so you can use `STREAMED_XOR_CHUNKS` mode and `SAMPLES` mode.
+Mimir supports both remote read mode, so you can use stream mode by setting `--remote-read-use-stream=true` flag.
 When you run Mimir, it exposes a port to serve HTTP on `8080 by default`.
 
 Next example of the local installation was in multi-tenant mode (3 instances of mimir) with nginx as load balancer.
diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml
index a659dd14b..6706ba967 100644
--- a/deployment/docker/docker-compose-cluster.yml
+++ b/deployment/docker/docker-compose-cluster.yml
@@ -2,7 +2,7 @@ version: '3.5'
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.93.0
+    image: victoriametrics/vmagent:v1.93.1
     depends_on:
       - "vminsert"
     ports:
@@ -32,7 +32,7 @@ services:
 
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.93.0-cluster
+    image: victoriametrics/vmstorage:v1.93.1-cluster
     ports:
       - 8482
       - 8400
@@ -44,7 +44,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.93.0-cluster
+    image: victoriametrics/vmstorage:v1.93.1-cluster
     ports:
       - 8482
       - 8400
@@ -56,7 +56,7 @@ services:
     restart: always
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.93.0-cluster
+    image: victoriametrics/vminsert:v1.93.1-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
     restart: always
   vmselect:
     container_name: vmselect
-    image: victoriametrics/vmselect:v1.93.0-cluster
+    image: victoriametrics/vmselect:v1.93.1-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -82,7 +82,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.93.0
+    image: victoriametrics/vmalert:v1.93.1
     depends_on:
       - "vmselect"
     ports:
diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
index dc4b90929..a1a59dd33 100644
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.5"
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.93.0
+    image: victoriametrics/vmagent:v1.93.1
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
     restart: always
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.93.0
+    image: victoriametrics/victoria-metrics:v1.93.1
     ports:
       - 8428:8428
       - 8089:8089
@@ -56,7 +56,7 @@ services:
     restart: always
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.93.0
+    image: victoriametrics/vmalert:v1.93.1
     depends_on:
       - "victoriametrics"
       - "alertmanager"
diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml
index 416046a35..6da6e164e 100644
--- a/deployment/logs-benchmark/docker-compose.yml
+++ b/deployment/logs-benchmark/docker-compose.yml
@@ -105,7 +105,7 @@ services:
       - '--config=/config.yml'
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.93.0
+    image: victoriametrics/victoria-metrics:v1.93.1
     ports:
       - '8428:8428'
     command:
diff --git a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
index 41fc7cd72..fdf6ed943 100644
--- a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
+++ b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
@@ -8,7 +8,7 @@
 4. Set variables `DIGITALOCEAN_API_TOKEN` with `VM_VERSION` for `packer` environment and run make from example below:
 
 ```console
-make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.93.0"
+make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.93.1"
 ```
 
 
diff --git a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
index e0765d5e2..8e8eeea1d 100755
--- a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
+++ b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
@@ -19,8 +19,8 @@ On the server:
   * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
 
 ********************************************************************************
-  # This image includes 1.93.0 version of VictoriaMetrics.
-  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0
+  # This image includes 1.93.1 version of VictoriaMetrics.
+  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.1
 
   # Welcome to VictoriaMetrics droplet!
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 35c5539bc..63cc8be2c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -25,22 +25,37 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 ## tip
 
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): add support for server-side copy of existing backups. See [these docs](https://docs.victoriametrics.com/vmbackup.html#server-side-copy-of-the-existing-backup) for details.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to set `member num` label for all the metrics scraped by a particular `vmagent` instance in [a cluster of vmagents](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) via `-promscrape.cluster.memberLabel` command-line flag. See [these docs](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly handle `unexpected EOF` error when parsing metrics in Prometheus exposition format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4817). 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): retry failed write request on the closed connection immediately, without waiting for backoff. This should improve data delivery speed and reduce amount of error logs emitted by vmagent when using idle connections. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): make the warning message more noticeable for text fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4848).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add button for auto-formatting PromQL/MetricsQL queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4681). Thanks to @aramattamara for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4694).
 
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently set UserAgent header to `vm_promscrape` during scraping with enabled or disabled `promscrape.streamParse`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4884) for details.
+
+## [v1.93.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.1)
+
+Released at 2023-08-23
+
+**v1.93.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.93.x line will be supported for at least 12 months since [v1.93.0](https://docs.victoriametrics.com/CHANGELOG.html#v1930) release**
+
+* BUGFIX: [storage](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: do not allow starting VictoriaMetrics components with improperly set boolean command-line flags in the form `-boolFlagName value`, since this leads to silent incomplete flags' parsing. This form should be replaced with `-boolFlagName=value`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4845).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly set labels from `-remoteWrite.label` command-line flag just before sending samples to the configured `-remoteWrite.url` according to [these docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics). Previously these labels were incorrectly set before [the relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) configured via `-remoteWrite.urlRelabelConfigs` and [the stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) configured via `-remoteWrite.streamAggr.config`, so these labels could be lost or incorrectly transformed before sending the samples to remote storage. The fix allows using `-remoteWrite.label` for identifying `vmagent` instances in [cluster mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247) and [these docs](https://docs.victoriametrics.com/stream-aggregation.html#cluster-mode) for more details.
 * BUGFIX: remove `DEBUG` logging when parsing `if` filters inside [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) and when parsing `match` filters inside [stream aggregation rules](https://docs.victoriametrics.com/stream-aggregation.html).
 * BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
-* BUGFIX: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): fix display of ingested rows rate for `Samples ingested/s` and `Samples rate` panels for vmagent's dasbhoard. Previously, not all ingested protocols were accounted in these panels. An extra panel `Rows rate` was added to `Ingestion` section to display the split for rows ingested rate by protocol.
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly re-use HTTP request object on `EOF` retries when querying the configured datasource. Previously, there was a small chance that query retry wouldn't succeed.
+
 
 ## [v1.93.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0)
 
 Released at 2023-08-12
 
+**It is recommended upgrading to [VictoriaMetrics v1.93.1](https://docs.victoriametrics.com/CHANGELOG.html#v1931) because v1.93.0 contains a bug, which could lead to incorrect indexdb rotation. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.**
+
 **v1.93.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
 The v1.93.x line will be supported for at least 12 months since [v1.93.0](https://docs.victoriametrics.com/CHANGELOG.html#v1930) release**
 
@@ -63,6 +78,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
 * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. 
 
+* BUGFIX: [storage](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): return human readable error if opentelemetry has json encoding. Follow-up after [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2570).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly validate scheme for `proxy_url` field at the scrape config. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4811) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816).
@@ -86,6 +102,9 @@ Released at 2023-07-28
 
 Released at 2023-07-27
 
+**Update note: this release contains backwards-incompatible change to indexdb,
+so rolling back to the previous versions of VictoriaMetrics may result in partial data loss of entries in indexdb.**
+
 **Update note**: starting from this release, [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) writes
 the following samples to the configured remote storage by default:
 
diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md
index 5b662c681..ff3be5a25 100644
--- a/docs/operator/CHANGELOG.md
+++ b/docs/operator/CHANGELOG.md
@@ -1,6 +1,13 @@
 # CHANGELOG
 
-## tip
+## Next release
+
+### Fixes
+
+### Features
+
+<a name="v0.36.0"></a>
+## [v0.36.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.36.0) - 23 Aug 2023
 
 ### Breaking changes
 
@@ -42,6 +49,8 @@
 - [vmoperator parameters](https://docs.victoriametrics.com/operator/vars.html): Add option `VM_ENABLESTRICTSECURITY` and enable strict security context by default. See [this issue](https://github.com/VictoriaMetrics/operator/issues/637), [this](https://github.com/VictoriaMetrics/operator/pull/692/) and [this](https://github.com/VictoriaMetrics/operator/pull/712) PR for details.
 - [vmoperator parameters](https://docs.victoriametrics.com/operator/vars.html): change option `VM_PSPAUTOCREATEENABLED` default value from `true` to `false` cause PodSecurityPolicy already got deprecated since [kubernetes v1.25](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#psp-v125). See [this pr](https://github.com/VictoriaMetrics/operator/pull/726) for details.
 
+[Changes][v0.36.0]
+
 <a name="v0.35.1"></a>
 ## [v0.35.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.35.1) - 12 Jul 2023
 
diff --git a/docs/operator/vars.md b/docs/operator/vars.md
index be3fd0952..2529047f2 100644
--- a/docs/operator/vars.md
+++ b/docs/operator/vars.md
@@ -10,7 +10,7 @@ aliases:
 - /operator/vars.html
 ---
 # Auto Generated vars for package config 
- updated at Fri Aug 18 07:32:53 UTC 2023 
+ updated at Wed Aug 23 17:26:30 UTC 2023 
 
 
 | varible name | variable default value | variable required | variable description |
@@ -20,7 +20,7 @@ aliases:
 | VM_CUSTOMCONFIGRELOADERIMAGE | victoriametrics/operator:config-reloader-v0.32.0 | false | - |
 | VM_PSPAUTOCREATEENABLED | false | false | - |
 | VM_VMALERTDEFAULT_IMAGE | victoriametrics/vmalert | false | - |
-| VM_VMALERTDEFAULT_VERSION | v1.91.3 | false | - |
+| VM_VMALERTDEFAULT_VERSION | v1.93.1 | false | - |
 | VM_VMALERTDEFAULT_PORT | 8080 | false | - |
 | VM_VMALERTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMALERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - |
@@ -31,7 +31,7 @@ aliases:
 | VM_VMALERTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
 | VM_VMALERTDEFAULT_CONFIGRELOADIMAGE | jimmidyson/configmap-reload:v0.3.0 | false | - |
 | VM_VMAGENTDEFAULT_IMAGE | victoriametrics/vmagent | false | - |
-| VM_VMAGENTDEFAULT_VERSION | v1.91.3 | false | - |
+| VM_VMAGENTDEFAULT_VERSION | v1.93.1 | false | - |
 | VM_VMAGENTDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.58.0 | false | - |
 | VM_VMAGENTDEFAULT_PORT | 8429 | false | - |
 | VM_VMAGENTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@@ -42,7 +42,7 @@ aliases:
 | VM_VMAGENTDEFAULT_CONFIGRELOADERCPU | 100m | false | - |
 | VM_VMAGENTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
 | VM_VMSINGLEDEFAULT_IMAGE | victoriametrics/victoria-metrics | false | - |
-| VM_VMSINGLEDEFAULT_VERSION | v1.91.3 | false | - |
+| VM_VMSINGLEDEFAULT_VERSION | v1.93.1 | false | - |
 | VM_VMSINGLEDEFAULT_PORT | 8429 | false | - |
 | VM_VMSINGLEDEFAULT_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMSINGLEDEFAULT_RESOURCE_LIMIT_MEM | 1500Mi | false | - |
@@ -53,14 +53,14 @@ aliases:
 | VM_VMSINGLEDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
 | VM_VMCLUSTERDEFAULT_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_IMAGE | victoriametrics/vmselect | false | - |
-| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.91.3-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.93.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_PORT | 8481 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_MEM | 1000Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_CPU | 100m | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_IMAGE | victoriametrics/vmstorage | false | - |
-| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.91.3-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.93.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMINSERTPORT | 8400 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMSELECTPORT | 8401 | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_PORT | 8482 | false | - |
@@ -69,7 +69,7 @@ aliases:
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_CPU | 250m | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_IMAGE | victoriametrics/vminsert | false | - |
-| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.91.3-cluster | false | - |
+| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.93.1-cluster | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_PORT | 8480 | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - |
 | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
@@ -88,7 +88,7 @@ aliases:
 | VM_VMALERTMANAGER_RESOURCE_REQUEST_CPU | 30m | false | - |
 | VM_DISABLESELFSERVICESCRAPECREATION | false | false | - |
 | VM_VMBACKUP_IMAGE | victoriametrics/vmbackupmanager | false | - |
-| VM_VMBACKUP_VERSION | v1.91.3-enterprise | false | - |
+| VM_VMBACKUP_VERSION | v1.93.1-enterprise | false | - |
 | VM_VMBACKUP_PORT | 8300 | false | - |
 | VM_VMBACKUP_USEDEFAULTRESOURCES | true | false | - |
 | VM_VMBACKUP_RESOURCE_LIMIT_MEM | 500Mi | false | - |
@@ -97,7 +97,7 @@ aliases:
 | VM_VMBACKUP_RESOURCE_REQUEST_CPU | 150m | false | - |
 | VM_VMBACKUP_LOGLEVEL | INFO | false | - |
 | VM_VMAUTHDEFAULT_IMAGE | victoriametrics/vmauth | false | - |
-| VM_VMAUTHDEFAULT_VERSION | v1.91.3 | false | - |
+| VM_VMAUTHDEFAULT_VERSION | v1.93.1 | false | - |
 | VM_VMAUTHDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1 | false | - |
 | VM_VMAUTHDEFAULT_PORT | 8427 | false | - |
 | VM_VMAUTHDEFAULT_USEDEFAULTRESOURCES | true | false | - |
diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index 80756a3f0..5970c5ce2 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -451,6 +451,12 @@ For example, see below time series produced by config with aggregation interval
 Aggregating irregular and sporadic metrics (received from [Lambdas](https://aws.amazon.com/lambda/)
 or [Cloud Functions](https://cloud.google.com/functions)) can be controlled via [staleness_inteval](#stream-aggregation-config).
 
+`increase` can be used as an alternative for [rate](https://docs.victoriametrics.com/MetricsQL.html#rate) function.
+For example, if we have `increase` with `interval` of `5m` for a counter `some_counter`, then to get `rate` we should divide
+the resulting aggregation by the `interval` in seconds: `some_counter:5m_increase / 300` is similar to `rate(some_counter[5m])`.
+Please note, opposite to [rate](https://docs.victoriametrics.com/MetricsQL.html#rate), `increase` aggregations can be 
+combined safely afterwards. This is helpful when the aggregation is calculated by more than one vmagent.
+
 ### count_series
 
 `count_series` counts the number of unique [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series).
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 41d14d650..aed6e84dd 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -765,10 +765,10 @@ as soon as it is parsed in stream parsing mode.
 
 A single `vmagent` instance can scrape tens of thousands of scrape targets. Sometimes this isn't enough due to limitations on CPU, network, RAM, etc.
 In this case scrape targets can be split among multiple `vmagent` instances (aka `vmagent` horizontal scaling, sharding and clustering).
-Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values.
-The flag value must be in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster.
-The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag. For example, the following commands
-spread scrape targets among a cluster of two `vmagent` instances:
+The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag.
+Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values
+in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster specified via `-promscrape.cluster.membersCount`.
+For example, the following commands spread scrape targets among a cluster of two `vmagent` instances:
 
 ```
 /path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.config=/path/to/config.yml ...
@@ -776,7 +776,7 @@ spread scrape targets among a cluster of two `vmagent` instances:
 ```
 
 The `-promscrape.cluster.memberNum` can be set to a StatefulSet pod name when `vmagent` runs in Kubernetes.
-The pod name must end with a number in the range `0 ... promscrape.cluster.memberNum-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
+The pod name must end with a number in the range `0 ... promscrape.cluster.membersCount-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
 
 By default, each scrape target is scraped only by a single `vmagent` instance in the cluster. If there is a need for replicating scrape targets among multiple `vmagent` instances,
 then `-promscrape.cluster.replicationFactor` command-line flag must be set to the desired number of replicas. For example, the following commands
@@ -792,6 +792,14 @@ If each target is scraped by multiple `vmagent` instances, then data deduplicati
 The `-dedup.minScrapeInterval` must be set to the `scrape_interval` configured at `-promscrape.config`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
+The `-promscrape.cluster.memberLabel` command-line flag allows specifying a name for `member num` label to add to all the scraped metrics.
+The value of the `member num` label is set to `-promscrape.cluster.memberNum`. For example, the following config instructs adding `vmagent_instance="0"` label
+to all the metrics scraped by the given `vmagent` instance:
+
+```
+/path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.cluster.memberLabel=vmagent_instance
+```
+
 See also [how to shard data among multiple remote storage systems](#sharding-among-remote-storages).
 
 
@@ -1387,8 +1395,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
   -promscrape.azureSDCheckInterval duration
      Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#azure_sd_configs for details (default 1m0s)
+  -promscrape.cluster.memberLabel string
+     If non-empty, then the label with this name and the -promscrape.cluster.memberNum value is added to all the scraped metrics
   -promscrape.cluster.memberNum string
-     The number of number in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
+     The number of vmagent instance in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. See also -promscrape.cluster.memberLabel (default "0")
   -promscrape.cluster.membersCount int
      The number of members in a cluster of scrapers. Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets
   -promscrape.cluster.name string
diff --git a/docs/vmanomaly.md b/docs/vmanomaly.md
index fda6ebc06..7c745c325 100644
--- a/docs/vmanomaly.md
+++ b/docs/vmanomaly.md
@@ -182,10 +182,10 @@ The license key can be passed via the following command-line flags:
   --license-file LICENSE_FILE
                         See https://victoriametrics.com/products/enterprise/
                         for trial license
-  --license-verify-offline LICENSE_VERIFY_OFFLINE
-                        Force offline verification of license code.
-                        License is verified online by default. This flag runs license verification
-                        offline.
+  --license-verify-offline {true,false}
+                        Force offline verification of license code. License is
+                        verified online by default. This flag runs license
+                        verification offline.
 ```
 
 Usage example:
diff --git a/docs/vmauth.md b/docs/vmauth.md
index fd03d86f2..e9f8bc1e1 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -232,6 +232,9 @@ ip_filters:
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
+Please note, vmauth doesn't follow redirects. If destination redirects request to a new location, make sure this 
+location is supported in vmauth `url_map` config.
+
 ## Security
 
 It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.
@@ -255,7 +258,7 @@ It is recommended protecting the following endpoints with authKeys:
 * `/metrics` with `metricsAuthKey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
 * `/debug/pprof` with `pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).
 
-`vmauth` also supports the ability to restict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
+`vmauth` also supports the ability to restrict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
 
 ## Monitoring
 
diff --git a/docs/vmctl.md b/docs/vmctl.md
index 88e5b10a6..236b7bb7e 100644
--- a/docs/vmctl.md
+++ b/docs/vmctl.md
@@ -22,6 +22,7 @@ Features:
 - migrate data from [Mimir](#migrating-data-from-mimir) to VictoriaMetrics
 - migrate data from [InfluxDB](#migrating-data-from-influxdb-1x) to VictoriaMetrics
 - migrate data from [OpenTSDB](#migrating-data-from-opentsdb) to VictoriaMetrics
+- migrate data from [Promscale](#migrating-data-from-promscale)
 - migrate data between [VictoriaMetrics](#migrating-data-from-victoriametrics) single or cluster version.
 - migrate data by [Prometheus remote read protocol](#migrating-data-by-remote-read-protocol) to VictoriaMetrics
 - [verify](#verifying-exported-blocks-from-victoriametrics) exported blocks from VictoriaMetrics single or cluster version.
@@ -317,6 +318,42 @@ Please see more about time filtering [here](https://docs.influxdata.com/influxdb
 Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
 You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
 
+## Migrating data from Promscale
+
+[Promscale](https://github.com/timescale/promscale) supports [Prometheus Remote Read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+Hence, `vmctl` can be used in [remote-read](https://docs.victoriametrics.com/vmctl.html#migrating-data-by-remote-read-protocol)
+mode to migrate historical data from Promscale to VictoriaMetrics.
+
+See the example of migration command below:
+```console
+./vmctl remote-read --remote-read-src-addr=http://127.0.0.1:9201/read --remote-read-step-interval=day --remote-read-use-stream=false --vm-addr=http://127.0.0.1:8428 --remote-read-filter-time-start=2023-08-21T00:00:00Z --vm-concurrency=6 --remote-read-disable-path-append=true
+Selected time range "2023-08-21 00:00:00 +0000 UTC" - "2023-08-21 14:11:41.561979 +0000 UTC" will be split into 1 ranges according to "day" step. Continue? [Y/n] y
+VM worker 0:↙ 82831 samples/s                                                                                                                                                                        
+VM worker 1:↙ 54378 samples/s                                                                                                                                                                        
+VM worker 2:↙ 121616 samples/s                                                                                                                                                                       
+VM worker 3:↙ 59164 samples/s                                                                                                                                                                        
+VM worker 4:↙ 59220 samples/s                                                                                                                                                                        
+VM worker 5:↙ 102072 samples/s                                                                                                                                                                       
+Processing ranges: 1 / 1 [██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/08/21 16:11:55 Import finished!
+2023/08/21 16:11:55 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 14.047045459s;
+  total samples: 262111;
+  samples/s: 18659.51;
+  total bytes: 5.3 MB;
+  bytes/s: 376.4 kB;
+  import requests: 6;
+  import requests retries: 0;
+2023/08/21 16:11:55 Total time: 14.063458792s
+```
+
+Here we specify the full path to Promscale's Remote Read API via `--remote-read-src-addr` and we disable auto-path
+appending via `--remote-read-disable-path-append` cmd-line flags. This is necessary, as Promscale has a different to
+Prometheus API path. Promscale doesn't support `STREAMED_XOR_CHUNKS` mode, so it may consume additional memory during
+the migration.
+
+
 ## Migrating data from Prometheus
 
 `vmctl` supports the `prometheus` mode for migrating data from Prometheus to VictoriaMetrics time-series database.
@@ -449,18 +486,28 @@ Found 2 blocks to import. Continue? [Y/n] y
 
 ## Migrating data by remote read protocol
 
-`vmctl` supports the `remote-read` mode for migrating data from databases which support 
-[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/)
+`vmctl` supports the `remote-read` mode for migrating data from databases which support
+[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+
+Remote read API implemented two modes:
+1. `SAMPLES` - returns a message that includes a list of raw samples (one protocol buffer message);
+2. `STREAMED_XOR_CHUNKS` - sending a set of small protocol buffer messages;
+   `STREAMED_XOR_CHUNKS` mode uses constant memory per request no matter how many samples was requested, when `SAMPLES` mode
+   buffered the whole response of the remote read a raw, uncompressed format in order to marshsal it in a potentially
+   huge protobuf message before sending it to the client. So potentially `SAMPLES` mode may lead to allocate a lot of memory
+   by both client and server each.
+   For more information, please check this [blog](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/).
 
 See `./vmctl remote-read --help` for details and full list of flags.
 
 To start the migration process configure the following flags:
 1. `--remote-read-src-addr` - data source address to read from;
-1. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
+2. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`,
    and for cluster version is equal to `--httpListenAddr` flag of vminsert component (for example `http://<vminsert>:8480/insert/<accountID>/prometheus`);
-1. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
-1. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
-1. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+3. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
+4. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
+5. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+6. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
 
 The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
@@ -527,7 +574,7 @@ and that you have a separate Thanos Store installation.
     - url: http://victoria-metrics:8428/api/v1/write
     ```
 
-1. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
+2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
     In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
 
     ```
@@ -535,7 +582,7 @@ and that you have a separate Thanos Store installation.
     2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2020_04" has been created
     ```
 
-1. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
+3. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
     storage.
 
 ### Historical data
@@ -544,12 +591,12 @@ Let's assume your data is stored on S3 served by minio. You first need to copy t
 then import it into VM using `vmctl` in `prometheus` mode.
 
 1. Copy data from minio.
-    1. Run the `minio/mc` Docker container.
-    1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
-    1. `mc cp -r minio/prometheus thanos-data`
-1. Import using `vmctl`.
-    1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
-    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
+    1.1 Run the `minio/mc` Docker container.
+    1.2 `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
+    1.3 `mc cp -r minio/prometheus thanos-data`
+2. Import using `vmctl`.
+    2.1 Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
+    2.2 Use [prometheus](#migrating-data-from-prometheus) mode to import data:
 
     ```
     vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
@@ -564,7 +611,7 @@ service (or anything that exposes gRPC StoreAPI e.g. Querier) via Prometheus rem
 If you want to migrate data, you should run [thanos-remote-read](https://github.com/G-Research/thanos-remote-read) proxy
 and define the Thanos store address `./thanos-remote-read -store 127.0.0.1:19194`. 
 It is important to know that `store` flag is Thanos Store API gRPC endpoint. 
-Also, it is important to know that thanos-remote-read proxy  doesn't support `STREAMED_XOR_CHUNKS` mode.
+Also, it is important to know that thanos-remote-read proxy doesn't support stream mode.
 When you run thanos-remote-read proxy, it exposes port to serve HTTP on `10080 by default`.
 
 The importing process example for local installation of Thanos
@@ -627,7 +674,7 @@ api:
 If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
 By default, Cortex uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
 
-It is important to know that Cortex doesn't support the `STREAMED_XOR_CHUNKS` mode.
+It is important to know that Cortex doesn't support the stream mode.
 When you run Cortex, it exposes a port to serve HTTP on `9009 by default`.
 
 The importing process example for the local installation of Cortex
@@ -685,7 +732,7 @@ api:
 If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
 By default, Mimir uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
 
-Mimir supports both remote read mode, so you can use `STREAMED_XOR_CHUNKS` mode and `SAMPLES` mode.
+Mimir supports both remote read mode, so you can use stream mode by setting `--remote-read-use-stream=true` flag.
 When you run Mimir, it exposes a port to serve HTTP on `8080 by default`.
 
 Next example of the local installation was in multi-tenant mode (3 instances of mimir) with nginx as load balancer.
diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go
index 420589dd9..3cec692dc 100644
--- a/lib/promscrape/client.go
+++ b/lib/promscrape/client.go
@@ -79,6 +79,8 @@ func concatTwoStrings(x, y string) string {
 	return s
 }
 
+const scrapeUserAgent = "vm_promscrape"
+
 func newClient(ctx context.Context, sw *ScrapeWork) *client {
 	var u fasthttp.URI
 	u.Update(sw.ScrapeURL)
@@ -120,8 +122,9 @@ func newClient(ctx context.Context, sw *ScrapeWork) *client {
 		logger.Fatalf("cannot create dial func: %s", err)
 	}
 	hc := &fasthttp.HostClient{
-		Addr:                         dialAddr,
-		Name:                         "vm_promscrape",
+		Addr: dialAddr,
+		// Name used in User-Agent request header
+		Name:                         scrapeUserAgent,
 		Dial:                         dialFunc,
 		IsTLS:                        isTLS,
 		TLSConfig:                    tlsCfg,
@@ -201,6 +204,7 @@ func (c *client) GetStreamReader() (*streamReader, error) {
 	// Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162
 	req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr)
+	req.Header.Set("User-Agent", scrapeUserAgent)
 	c.setHeaders(req)
 	c.setProxyHeaders(req)
 	scrapeRequests.Inc()
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 5cf006c00..9ac356b7b 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -56,9 +56,12 @@ var (
 	clusterMembersCount = flag.Int("promscrape.cluster.membersCount", 0, "The number of members in a cluster of scrapers. "+
 		"Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . "+
 		"Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets")
-	clusterMemberNum = flag.String("promscrape.cluster.memberNum", "0", "The number of number in the cluster of scrapers. "+
+	clusterMemberNum = flag.String("promscrape.cluster.memberNum", "0", "The number of vmagent instance in the cluster of scrapers. "+
 		"It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. "+
-		"Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name")
+		"Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. "+
+		"See also -promscrape.cluster.memberLabel")
+	clusterMemberLabel = flag.String("promscrape.cluster.memberLabel", "", "If non-empty, then the label with this name and the -promscrape.cluster.memberNum value "+
+		"is added to all the scraped metrics")
 	clusterReplicationFactor = flag.Int("promscrape.cluster.replicationFactor", 1, "The number of members in the cluster, which scrape the same targets. "+
 		"If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication")
 	clusterName = flag.String("promscrape.cluster.name", "", "Optional name of the cluster. If multiple vmagent clusters scrape the same targets, "+
@@ -80,6 +83,10 @@ func mustInitClusterMemberID() {
 	if err != nil {
 		logger.Fatalf("cannot parse -promscrape.cluster.memberNum=%q: %s", *clusterMemberNum, err)
 	}
+	if n < 0 || n >= *clusterMembersCount {
+		logger.Fatalf("-promscrape.cluster.memberNum must be in the range [0..%d] according to -promscrape.cluster.membersCount=%d",
+			*clusterMembersCount, *clusterMembersCount)
+	}
 	clusterMemberID = n
 }
 
@@ -1337,6 +1344,9 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 	if labels.Get("instance") == "" {
 		labels.Add("instance", address)
 	}
+	if *clusterMemberLabel != "" && *clusterMemberNum != "" {
+		labels.Add(*clusterMemberLabel, *clusterMemberNum)
+	}
 	// Remove references to deleted labels, so GC could clean strings for label name and label value past len(labels.Labels).
 	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details.
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index 5ef0a8290..e61c0f0c0 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -757,7 +757,7 @@ func (s *Storage) mustRotateIndexDB(currentTime time.Time) {
 	idbNew := mustOpenIndexDB(idbNewPath, s, &s.isReadOnly)
 
 	// Update nextRotationTimestamp
-	nextRotationTimestamp := currentTime.UnixMilli() + s.retentionMsecs/1000
+	nextRotationTimestamp := currentTime.Unix() + s.retentionMsecs/1000
 	atomic.StoreInt64(&s.nextRotationTimestamp, nextRotationTimestamp)
 
 	// Set idbNext to idbNew


Test patch:
diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go
index 68c2f6e8f..7b06f42db 100644
--- a/app/vmalert/datasource/vm_test.go
+++ b/app/vmalert/datasource/vm_test.go
@@ -629,10 +629,7 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequestPOST()
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
+			req := tc.vm.newRequest()
 			switch tc.vm.dataSourceType {
 			case "", datasourcePrometheus:
 				if tc.queryRange {
@@ -727,10 +724,7 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req, err := vm.newRequestPOST()
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
+			req := vm.newQueryRequest("foo", time.Now())
 			tt.checkFn(t, req)
 		})
 	}