Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4914 (2023-08-29 12:16:25)

vmauth: added ability to set and remove response headers (#4825)
issue #4825 

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Patch coverage: **`23.52%`** and project coverage change: **`+0.02%`** :tada:
> Comparison is base [(`317a273`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/317a273c6ddece0cd864da2d8d42f40d73a70d6a?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.53% compared to head [(`8ec486a`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.55%.
> Report is 7 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #4914      +/-   ##
==========================================
+ Coverage   60.53%   60.55%   +0.02%     
==========================================
  Files         384      384              
  Lines       72300    72306       +6     
==========================================
+ Hits        43766    43787      +21     
+ Misses      26134    26122      -12     
+ Partials     2400     2397       -3     
```


| [Files Changed](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Coverage Δ | |
|---|---|---|
| [app/vmauth/auth\_config.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC9hdXRoX2NvbmZpZy5nbw==) | `56.63% <ø> (ø)` | |
| [app/vmauth/main.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC9tYWluLmdv) | `0.00% <0.00%> (ø)` | |
| [app/vmauth/target\_url.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC90YXJnZXRfdXJsLmdv) | `86.95% <100.00%> (ø)` | |

... and [11 files with indirect coverage changes](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)


</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4914?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vlinsert/loki/loki.go b/app/vlinsert/loki/loki.go
index 878616328..09faa742c 100644
--- a/app/vlinsert/loki/loki.go
+++ b/app/vlinsert/loki/loki.go
@@ -3,9 +3,10 @@ package loki
 import (
 	"net/http"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -14,12 +15,22 @@ var (
 )
 
 // RequestHandler processes Loki insert requests
-//
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	if path != "/api/v1/push" {
+	switch path {
+	case "/api/v1/push":
+		return handleInsert(r, w)
+	case "/ready":
+		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ready"))
+		return true
+	default:
 		return false
 	}
+}
+
+// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+func handleInsert(r *http.Request, w http.ResponseWriter) bool {
 	contentType := r.Header.Get("Content-Type")
 	switch contentType {
 	case "application/json":
diff --git a/app/vmauth/README.md b/app/vmauth/README.md
index dea54e0e8..ea14aeff6 100644
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -117,11 +117,16 @@ users:
 
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # The `X-Server-Hostname` http header is removed from the proxied response.
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
 - bearer_token: "YYY"
   url_prefix: "http://localhost:8428"
+  # extra headers to add to the request or remove from the request (if header value is empty)
   headers:
-  - "X-Scope-OrgID: foobar"
+    - "X-Scope-OrgID: foobar"
+  # extra headers to add to the response or remove from the response (if header value is empty)
+  response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
 
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are proxied to http://localhost:8428 .
@@ -175,6 +180,7 @@ users:
   #
   # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
   #   The "X-Scope-OrgID: abc" http header is added to these requests.
+  #   The "X-Server-Hostname" http header is removed from the proxied response.
   #
   # Request which do not match `src_paths` from the `url_map` are proxied to the urls from `default_url`
   # in a round-robin manner. The original request path is passed in `request_path` query arg.
@@ -194,6 +200,8 @@ users:
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
     - "X-Scope-OrgID: abc"
+    response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
     ip_filters:
       deny_list: [127.0.0.1]
   default_url:
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index 6fab91dc2..af1b0573f 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -37,15 +37,15 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name                  string     `yaml:"name,omitempty"`
-	BearerToken           string     `yaml:"bearer_token,omitempty"`
-	Username              string     `yaml:"username,omitempty"`
-	Password              string     `yaml:"password,omitempty"`
-	URLPrefix             *URLPrefix `yaml:"url_prefix,omitempty"`
-	URLMaps               []URLMap   `yaml:"url_map,omitempty"`
-	Headers               []Header   `yaml:"headers,omitempty"`
-	MaxConcurrentRequests int        `yaml:"max_concurrent_requests,omitempty"`
-	DefaultURL            *URLPrefix `yaml:"default_url,omitempty"`
+	Name                  string      `yaml:"name,omitempty"`
+	BearerToken           string      `yaml:"bearer_token,omitempty"`
+	Username              string      `yaml:"username,omitempty"`
+	Password              string      `yaml:"password,omitempty"`
+	URLPrefix             *URLPrefix  `yaml:"url_prefix,omitempty"`
+	URLMaps               []URLMap    `yaml:"url_map,omitempty"`
+	HeadersConf           HeadersConf `yaml:",inline"`
+	MaxConcurrentRequests int         `yaml:"max_concurrent_requests,omitempty"`
+	DefaultURL            *URLPrefix  `yaml:"default_url,omitempty"`
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
@@ -54,6 +54,12 @@ type UserInfo struct {
 	requestsDuration *metrics.Summary
 }
 
+// HeadersConf represents config for request and response headers.
+type HeadersConf struct {
+	RequestHeaders  []Header `yaml:"headers,omitempty"`
+	ResponseHeaders []Header `yaml:"response_headers,omitempty"`
+}
+
 func (ui *UserInfo) beginConcurrencyLimit() error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
@@ -105,9 +111,9 @@ func (h *Header) MarshalYAML() (interface{}, error) {
 
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
-	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
-	Headers   []Header   `yaml:"headers,omitempty"`
+	SrcPaths    []*SrcPath  `yaml:"src_paths,omitempty"`
+	URLPrefix   *URLPrefix  `yaml:"url_prefix,omitempty"`
+	HeadersConf HeadersConf `yaml:",inline"`
 }
 
 // SrcPath represents an src path
diff --git a/app/vmauth/example_config.yml b/app/vmauth/example_config.yml
index e7721be0e..f9e149626 100644
--- a/app/vmauth/example_config.yml
+++ b/app/vmauth/example_config.yml
@@ -12,11 +12,16 @@ users:
 
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # The `X-Server-Hostname:` http header is removed from the proxied response.
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
 - bearer_token: "YYY"
   url_prefix: "http://localhost:8428"
+  # extra headers to add to the request or remove from the request (if header value is empty)
   headers:
   - "X-Scope-OrgID: foobar"
+  # extra headers to add to the response or remove from the response (if header value is empty)
+  response_headers:
+  - "X-Server-Hostname:" # empty value means the header will be removed from the response
 
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are proxied to http://localhost:8428 .
diff --git a/app/vmauth/main.go b/app/vmauth/main.go
index 925bfb7c6..1f7d685a0 100644
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -151,7 +151,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
-	up, headers := ui.getURLPrefixAndHeaders(u)
+	up, headersConf := ui.getURLPrefixAndHeaders(u)
 	isDefault := false
 	if up == nil {
 		missingRouteRequests.Inc()
@@ -159,7 +159,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			httpserver.Errorf(w, r, "missing route for %q", u.String())
 			return
 		}
-		up, headers = ui.DefaultURL, ui.Headers
+		up, headersConf = ui.DefaultURL, ui.HeadersConf
 		isDefault = true
 	}
 	r.Body = &readTrackingBody{
@@ -178,7 +178,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, headers)
+		ok := tryProcessingRequest(w, r, targetURL, headersConf)
 		bu.put()
 		if ok {
 			return
@@ -192,13 +192,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headersConf HeadersConf) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
+	updateHeadersByConfig(req.Header, headersConf.RequestHeaders)
 	transportOnce.Do(transportInit)
 	res, err := transport.RoundTrip(req)
 	if err != nil {
@@ -222,6 +220,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
+	updateHeadersByConfig(w.Header(), headersConf.ResponseHeaders)
 	w.WriteHeader(res.StatusCode)
 
 	copyBuf := copyBufPool.Get()
@@ -247,6 +246,16 @@ func copyHeader(dst, src http.Header) {
 	}
 }
 
+func updateHeadersByConfig(headers http.Header, config []Header) {
+	for _, h := range config {
+		if h.Value == "" {
+			headers.Del(h.Name)
+		} else {
+			headers.Set(h.Name, h.Value)
+		}
+	}
+}
+
 func sanitizeRequestHeaders(r *http.Request) *http.Request {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := r.Clone(r.Context())
diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go
index 7e4afd24a..365484d95 100644
--- a/app/vmauth/target_url.go
+++ b/app/vmauth/target_url.go
@@ -29,18 +29,18 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf) {
 	for _, e := range ui.URLMaps {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
-				return e.URLPrefix, e.Headers
+				return e.URLPrefix, e.HeadersConf
 			}
 		}
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix, ui.Headers
+		return ui.URLPrefix, ui.HeadersConf
 	}
-	return nil, nil
+	return nil, HeadersConf{}
 }
 
 func normalizeURL(uOrig *url.URL) *url.URL {
diff --git a/app/vmctl/README.md b/app/vmctl/README.md
index 4cee35e00..62f142394 100644
--- a/app/vmctl/README.md
+++ b/app/vmctl/README.md
@@ -489,13 +489,14 @@ doesn't support it).
 See `./vmctl remote-read --help` for details and full list of flags.
 
 To start the migration process configure the following flags:
+
 1. `--remote-read-src-addr` - data source address to read from;
-2. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
+1. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
    and for cluster version is equal to `--httpListenAddr` flag of vminsert component (for example `http://<vminsert>:8480/insert/<accountID>/prometheus`);
-3. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
-4. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
-5. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
-6. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
+1. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
+1. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
+1. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+1. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
 
 The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
@@ -562,7 +563,7 @@ and that you have a separate Thanos Store installation.
     - url: http://victoria-metrics:8428/api/v1/write
     ```
 
-2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
+1. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
     In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
 
     ```
@@ -570,7 +571,7 @@ and that you have a separate Thanos Store installation.
     2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2020_04" has been created
     ```
 
-3. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
+1. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
     storage.
 
 ### Historical data
@@ -579,12 +580,13 @@ Let's assume your data is stored on S3 served by minio. You first need to copy t
 then import it into VM using `vmctl` in `prometheus` mode.
 
 1. Copy data from minio.
-    1.1 Run the `minio/mc` Docker container.
-    1.2 `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
-    1.3 `mc cp -r minio/prometheus thanos-data`
-2. Import using `vmctl`.
-    2.1 Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
-    2.2 Use [prometheus](#migrating-data-from-prometheus) mode to import data:
+    1. Run the `minio/mc` Docker container.
+    1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
+    1. `mc cp -r minio/prometheus thanos-data`
+ 
+1. Import using `vmctl`.
+    1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
+    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
 
     ```
     vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index b372cfc38..cd905b99f 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -80,6 +80,7 @@ publish-via-docker: \
 		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
 		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
 		-o type=image \
+		--provenance=false \
 		-f app/$(APP_NAME)/multiarch/Dockerfile \
 		--push \
 		bin
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3279d21f3..df5e0983e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,21 +24,27 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 
 ## tip
 
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add support for numbers with underscore delimiters such as `1_234_567_890` and `1.234_567_890`. These numbers are easier to read than `1234567890` and `1.234567890`.
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): add support for server-side copy of existing backups. See [these docs](https://docs.victoriametrics.com/vmbackup.html#server-side-copy-of-the-existing-backup) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to set `member num` label for all the metrics scraped by a particular `vmagent` instance in [a cluster of vmagents](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) via `-promscrape.cluster.memberLabel` command-line flag. See [these docs](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4247).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not log `unexpected EOF` when reading incoming metrics, since this error is expected and is handled during metrics' parsing. This reduces the amounts of noisy logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4817).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): retry failed write request on the closed connection immediately, without waiting for backoff. This should improve data delivery speed and reduce amount of error logs emitted by vmagent when using idle connections. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduces load on Kubernetes control plane during initial service discovery. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4855) for details.
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce the maximum recovery time at `vmselect` and `vminsert` when some of `vmstorage` nodes become unavailable because of networking issues from 60 seconds to 3 seconds by default. The recovery time can be tuned at `vmselect` and `vminsert` nodes with `-vmstorageUserTimeout` command-line flag if needed. Thanks to @wjordan for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4423).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): make the warning message more noticeable for text fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4848).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add button for auto-formatting PromQL/MetricsQL queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4681). Thanks to @aramattamara for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4694).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve accessibility score to 100 according to [Google's Lighthouse](https://developer.chrome.com/docs/lighthouse/accessibility/) tests.
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): organize `min`, `max`, `median` values on the chart legend and tooltips for better visibility.
 * FEATURE: dashboards: provide copies of Grafana dashboards alternated with VictoriaMetrics datasource at [dashboards/vm](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards/vm).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): added ability to set and clear response headers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4825) and [these docs](https://docs.victoriametrics.com/vmauth.html#auth-config) for details.
 
+* BUGFIX: [build](https://docs.victoriametrics.com/): fix Docker builds for old Docker releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4907).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently set `User-Agent` header to `vm_promscrape` during scraping with enabled or disabled [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4884).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently set timeout for scraping with enabled or disabled [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4847).
 * BUGFIX: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): fix display of ingested rows rate for `Samples ingested/s` and `Samples rate` panels for vmagent's dasbhoard. Previously, not all ingested protocols were accounted in these panels. An extra panel `Rows rate` was added to `Ingestion` section to display the split for rows ingested rate by protocol.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly re-use HTTP request object on `EOF` retries when querying the configured datasource. Previously, there was a small chance that query retry wouldn't succeed.
+* BUGFIX: [vmselect](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): correctly handle requests with `/select/multitenant` prefix. Such requests must be rejected since vmselect does not support multitenancy endpoint. Previously, such requests were causing panic. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4910).
+* BUGFIX: [vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly check for [read-only state](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode) at `vmstorage`. Previously it wasn't properly checked, which could lead to increased resource usage and data ingestion slowdown when some of `vmstorage` nodes are in read-only mode. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4870).
 
 ## [v1.93.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.1)
 
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index e5106fe6c..81df16782 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -67,6 +67,7 @@ The list of MetricsQL features on top of PromQL:
   depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)).
   For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
   It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
+* Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
   selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index a72bf3bae..e4977cc51 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -11,6 +11,7 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
   * `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
 
 * BUGFIX: fix possible panic when no data is written to VictoriaLogs for a long time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4895). Thanks to @crossoverJie for filing and fixing the issue.
+* BUGFIX: add `/insert/loky/ready` endpoint, which is used by Promtail for healthchecks. This should remove `unsupported path requested: /insert/loki/ready` warning logs. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4762#issuecomment-1690966722).
 
 ## [v0.3.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.3.0-victorialogs)
 
diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md
index e751c23d3..c4059f233 100644
--- a/docs/operator/CHANGELOG.md
+++ b/docs/operator/CHANGELOG.md
@@ -3,7 +3,6 @@
 ## Next release
 
 ### Fixes
-- [vmselect](https://docs.victoriametrics.com/operator/api.html#vmcluster): fix cache directory when `cacheDataPath` not specified, before it will use `/tmp` which is protect by default strict securityContext.
 
 - TODO
 
@@ -11,6 +10,17 @@
 
 - TODO
 
+<a name="v0.37.0"></a>
+## [v0.37.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.37.0) - 30 Aug 2023
+
+### Fixes
+
+- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): fix unmarshalling for streaming aggregation `match` field.
+
+### Features
+
+- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): support [multiple if conditions](https://docs.victoriametrics.com/vmagent.html#relabeling:~:text=the%20if%20option%20may%20contain%20more%20than%20one%20filter) for relabeling. See [this issue](https://github.com/VictoriaMetrics/operator/issues/730) for details. 
+
 <a name="v0.36.1"></a>
 ## [v0.36.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.36.0) - 25 Aug 2023
 
diff --git a/docs/operator/api.md b/docs/operator/api.md
index b154de79c..66973cb06 100644
--- a/docs/operator/api.md
+++ b/docs/operator/api.md
@@ -996,7 +996,7 @@ StreamAggrRule defines the rule in stream aggregation config
 
 | Field | Description | Scheme | Required |
 | ----- | ----------- | ------ | -------- |
-| match | Match is a label selector for filtering time series for the given selector.\n\nIf the match isn&#39;t set, then all the input time series are processed. | Match | false |
+| match | Match is a label selector (or list of label selectors) for filtering time series for the given selector.\n\nIf the match isn&#39;t set, then all the input time series are processed. | StringOrArray | false |
 | interval | Interval is the interval between aggregations. | string | true |
 | staleness_interval | StalenessInterval defines an interval after which the series state will be reset if no samples have been sent during it. | string | false |
 | outputs | Outputs is a list of output aggregate functions to produce.\n\nThe following names are allowed:\n\n- total - aggregates input counters - increase - counts the increase over input counters - count_series - counts the input series - count_samples - counts the input samples - sum_samples - sums the input samples - last - the last biggest sample value - min - the minimum sample value - max - the maximum sample value - avg - the average value across all the samples - stddev - standard deviation across all the samples - stdvar - standard variance across all the samples - histogram_bucket - creates VictoriaMetrics histogram for input samples - quantiles(phi1, ..., phiN) - quantiles&#39; estimation for phi in the range [0..1]\n\nThe output time series will have the following names:\n\n  input_name:aggr_&lt;interval&gt;_&lt;output&gt; | []string | true |
@@ -1461,7 +1461,7 @@ RelabelConfig allows dynamic rewriting of the label set, being applied to sample
 | modulus | Modulus to take of the hash of the source label values. | uint64 | false |
 | replacement | Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is &#39;$1&#39; | string | false |
 | action | Action to perform based on regex matching. Default is &#39;replace&#39; | string | false |
-| if | If represents metricsQL match expression: &#39;{__name__=~\&#34;foo_.*\&#34;}&#39; | string | false |
+| if | If represents metricsQL match expression (or list of expressions): &#39;{__name__=~\&#34;foo_.*\&#34;}&#39; | StringOrArray | false |
 | match | Match is used together with Labels for `action: graphite` | string | false |
 | labels | Labels is used together with Match for `action: graphite` | map[string]string | false |
 
diff --git a/docs/operator/vars.md b/docs/operator/vars.md
index 7c86c43c2..f201980b5 100644
--- a/docs/operator/vars.md
+++ b/docs/operator/vars.md
@@ -10,7 +10,7 @@ aliases:
 - /operator/vars.html
 ---
 # Auto Generated vars for package config 
- updated at Fri Aug 25 13:22:16 UTC 2023 
+ updated at Wed Aug 30 10:13:42 UTC 2023 
 
 
 | varible name | variable default value | variable required | variable description |
diff --git a/docs/vmauth.md b/docs/vmauth.md
index 4dd06f501..e8f4cb243 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -128,11 +128,16 @@ users:
 
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # The `X-Server-Hostname` http header is removed from the proxied response.
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
 - bearer_token: "YYY"
   url_prefix: "http://localhost:8428"
+  # extra headers to add to the request or remove from the request (if header value is empty)
   headers:
-  - "X-Scope-OrgID: foobar"
+    - "X-Scope-OrgID: foobar"
+  # extra headers to add to the response or remove from the response (if header value is empty)
+  response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
 
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are proxied to http://localhost:8428 .
@@ -186,6 +191,7 @@ users:
   #
   # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
   #   The "X-Scope-OrgID: abc" http header is added to these requests.
+  #   The "X-Server-Hostname" http header is removed from the proxied response.
   #
   # Request which do not match `src_paths` from the `url_map` are proxied to the urls from `default_url`
   # in a round-robin manner. The original request path is passed in `request_path` query arg.
@@ -205,6 +211,8 @@ users:
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
     - "X-Scope-OrgID: abc"
+    response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
     ip_filters:
       deny_list: [127.0.0.1]
   default_url:
diff --git a/docs/vmctl.md b/docs/vmctl.md
index 1a3b666d4..9e71165b8 100644
--- a/docs/vmctl.md
+++ b/docs/vmctl.md
@@ -500,13 +500,14 @@ doesn't support it).
 See `./vmctl remote-read --help` for details and full list of flags.
 
 To start the migration process configure the following flags:
+
 1. `--remote-read-src-addr` - data source address to read from;
-2. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
+1. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
    and for cluster version is equal to `--httpListenAddr` flag of vminsert component (for example `http://<vminsert>:8480/insert/<accountID>/prometheus`);
-3. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
-4. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
-5. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
-6. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
+1. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
+1. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
+1. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+1. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
 
 The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
@@ -573,7 +574,7 @@ and that you have a separate Thanos Store installation.
     - url: http://victoria-metrics:8428/api/v1/write
     ```
 
-2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
+1. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
     In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
 
     ```
@@ -581,7 +582,7 @@ and that you have a separate Thanos Store installation.
     2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2020_04" has been created
     ```
 
-3. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
+1. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
     storage.
 
 ### Historical data
@@ -590,12 +591,13 @@ Let's assume your data is stored on S3 served by minio. You first need to copy t
 then import it into VM using `vmctl` in `prometheus` mode.
 
 1. Copy data from minio.
-    1.1 Run the `minio/mc` Docker container.
-    1.2 `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
-    1.3 `mc cp -r minio/prometheus thanos-data`
-2. Import using `vmctl`.
-    2.1 Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
-    2.2 Use [prometheus](#migrating-data-from-prometheus) mode to import data:
+    1. Run the `minio/mc` Docker container.
+    1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
+    1. `mc cp -r minio/prometheus thanos-data`
+
+1. Import using `vmctl`.
+    1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
+    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
 
     ```
     vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
diff --git a/go.mod b/go.mod
index 7b256e6c3..4b4181414 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.2.0
 	github.com/VictoriaMetrics/metrics v1.24.0
-	github.com/VictoriaMetrics/metricsql v0.63.0
+	github.com/VictoriaMetrics/metricsql v0.64.0
 	github.com/aws/aws-sdk-go-v2 v1.21.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.37
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.81
diff --git a/go.sum b/go.sum
index fbfd2b688..bdba8f831 100644
--- a/go.sum
+++ b/go.sum
@@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b
 github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE=
 github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
 github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
-github.com/VictoriaMetrics/metricsql v0.63.0 h1:RRu3lln7uhQwSRkzAknOUyB0uP9LwymFMHnzDqGbZ40=
-github.com/VictoriaMetrics/metricsql v0.63.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
+github.com/VictoriaMetrics/metricsql v0.64.0 h1:uty6AXQFY3OpQ+eopo1jDjCcTctuqkqYLnRbQVhukW8=
+github.com/VictoriaMetrics/metricsql v0.64.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
diff --git a/lib/auth/auth.go b/lib/auth/auth.go
index 38e235448..9a7a4346f 100644
--- a/lib/auth/auth.go
+++ b/lib/auth/auth.go
@@ -24,12 +24,7 @@ func (t *Token) String() string {
 }
 
 // NewToken returns new Token for the given authToken.
-//
-// If authToken == "multitenant", then nil Token is returned.
 func NewToken(authToken string) (*Token, error) {
-	if authToken == "multitenant" {
-		return nil, nil
-	}
 	var t Token
 	if err := t.Init(authToken); err != nil {
 		return nil, err
@@ -37,6 +32,16 @@ func NewToken(authToken string) (*Token, error) {
 	return &t, nil
 }
 
+// NewTokenPossibleMultitenant returns new Token for the given authToken.
+//
+// If authToken == "multitenant", then nil Token is returned.
+func NewTokenPossibleMultitenant(authToken string) (*Token, error) {
+	if authToken == "multitenant" {
+		return nil, nil
+	}
+	return NewToken(authToken)
+}
+
 // Init initializes t from authToken.
 func (t *Token) Init(authToken string) error {
 	tmp := strings.Split(authToken, ":")
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index 7732847e5..78bb7ddd7 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -580,7 +580,13 @@ func (uw *urlWatcher) reloadObjects() string {
 	}
 
 	startTime := time.Now()
-	requestURL := uw.apiURL
+	apiURL := uw.apiURL
+
+	// Set resourceVersion to 0 in order to reduce load on Kubernetes control plane.
+	// See https://kubernetes.io/docs/reference/using-api/api-concepts/#semantics-for-get-and-list
+	// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4855 .
+	delimiter := getQueryArgsDelimiter(apiURL)
+	requestURL := apiURL + delimiter + "resourceVersion=0&resourceVersionMatch=NotOlderThan"
 	resp, err := uw.gw.doRequest(requestURL)
 	if err != nil {
 		logger.Errorf("cannot perform request to %q: %s", requestURL, err)
@@ -657,10 +663,7 @@ func (uw *urlWatcher) watchForUpdates() {
 		}
 	}
 	apiURL := uw.apiURL
-	delimiter := "?"
-	if strings.Contains(apiURL, "?") {
-		delimiter = "&"
-	}
+	delimiter := getQueryArgsDelimiter(apiURL)
 	timeoutSeconds := time.Duration(0.9 * float64(uw.gw.client.Timeout)).Seconds()
 	apiURL += delimiter + "watch=1&allowWatchBookmarks=true&timeoutSeconds=" + strconv.Itoa(int(timeoutSeconds))
 	for {
@@ -943,3 +946,10 @@ func getObjectParsersForRole(role string) (parseObjectFunc, parseObjectListFunc)
 		return nil, nil
 	}
 }
+
+func getQueryArgsDelimiter(apiURL string) string {
+	if strings.Contains(apiURL, "?") {
+		return "&"
+	}
+	return "?"
+}
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
index f0a2a3466..da6a47e68 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
@@ -233,7 +233,7 @@ func scanPositiveNumber(s string) (string, error) {
 		}
 		return s[:i], nil
 	}
-	for i < len(s) && isDecimalChar(s[i]) {
+	for i < len(s) && isDecimalCharOrUnderscore(s[i]) {
 		i++
 	}
 
@@ -258,7 +258,7 @@ func scanPositiveNumber(s string) (string, error) {
 		// Scan fractional part. It cannot be empty.
 		i++
 		j := i
-		for j < len(s) && isDecimalChar(s[j]) {
+		for j < len(s) && isDecimalCharOrUnderscore(s[j]) {
 			j++
 		}
 		i = j
@@ -673,6 +673,10 @@ func isDecimalChar(ch byte) bool {
 	return ch >= '0' && ch <= '9'
 }
 
+func isDecimalCharOrUnderscore(ch byte) bool {
+	return isDecimalChar(ch) || ch == '_'
+}
+
 func isHexChar(ch byte) bool {
 	return isDecimalChar(ch) || ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F'
 }
diff --git a/vendor/modules.txt b/vendor/modules.txt
index c95f6db45..29b1bdb91 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.24.0
 ## explicit; go 1.20
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.63.0
+# github.com/VictoriaMetrics/metricsql v0.64.0
 ## explicit; go 1.13
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop


Test patch:
diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go
index e794231aa..dc465bcfc 100644
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@@ -299,14 +299,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -325,14 +327,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -389,14 +393,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -419,14 +425,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go
index 482138e17..41c4ea1af 100644
--- a/app/vmauth/target_url_test.go
+++ b/app/vmauth/target_url_test.go
@@ -24,7 +24,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
-		headersStr := fmt.Sprintf("%q", headers)
+		headersStr := fmt.Sprintf("%q", headers.RequestHeaders)
 		if headersStr != expectedHeaders {
 			t.Fatalf("unexpected headers; got %s; want %s", headersStr, expectedHeaders)
 		}
@@ -35,10 +35,10 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	}, "", "http://foo.bar/.", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-		Headers: []Header{{
+		HeadersConf: HeadersConf{RequestHeaders: []Header{{
 			Name:  "bb",
 			Value: "aaa",
-		}},
+		}}},
 	}, "/", "http://foo.bar", `[{"bb" "aaa"}]`)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar/federate"),
@@ -62,7 +62,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
-				Headers: []Header{
+				HeadersConf: HeadersConf{RequestHeaders: []Header{
 					{
 						Name:  "xx",
 						Value: "aa",
@@ -71,7 +71,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 						Name:  "yy",
 						Value: "asdf",
 					},
-				},
+				}},
 			},
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
@@ -79,10 +79,10 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
-		Headers: []Header{{
+		HeadersConf: HeadersConf{RequestHeaders: []Header{{
 			Name:  "bb",
 			Value: "aaa",
-		}},
+		}}},
 	}
 	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`)
 	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
@@ -128,7 +128,10 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if up != nil {
 			t.Fatalf("unexpected non-empty up=%#v", up)
 		}
-		if headers != nil {
+		if headers.RequestHeaders != nil {
+			t.Fatalf("unexpected non-empty headers=%q", headers)
+		}
+		if headers.ResponseHeaders != nil {
 			t.Fatalf("unexpected non-empty headers=%q", headers)
 		}
 	}
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index b39887bea..90d31e349 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -86,6 +86,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("int_with_underscores", func(t *testing.T) {
+		t.Parallel()
+		q := `123_456_789`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{123456789, 123456789, 123456789, 123456789, 123456789, 123456789},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("float_with_underscores", func(t *testing.T) {
+		t.Parallel()
+		q := `1_2.3_456_789`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12.3456789, 12.3456789, 12.3456789, 12.3456789, 12.3456789, 12.3456789},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("duration-constant", func(t *testing.T) {
 		t.Parallel()
 		q := `1h23m5S`
@@ -141,6 +163,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("num-with-suffix-5", func(t *testing.T) {
+		t.Parallel()
+		q := `1_234M`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1234e6, 1234e6, 1234e6, 1234e6, 1234e6, 1234e6},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("simple-arithmetic", func(t *testing.T) {
 		t.Parallel()
 		q := `-1+2 *3 ^ 4+5%6`
diff --git a/lib/auth/auth_test.go b/lib/auth/auth_test.go
index eddf1c6fe..49f8ca765 100644
--- a/lib/auth/auth_test.go
+++ b/lib/auth/auth_test.go
@@ -26,6 +26,24 @@ func TestNewTokenSuccess(t *testing.T) {
 	f("1:4294967295", "1:4294967295")
 	// max uint32 accountID and projectID
 	f("4294967295:4294967295", "4294967295:4294967295")
+}
+
+func TestNewTokenPossibleMultitenantSuccess(t *testing.T) {
+	f := func(token string, want string) {
+		t.Helper()
+		newToken, err := NewTokenPossibleMultitenant(token)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		got := newToken.String()
+		if got != want {
+			t.Fatalf("unexpected NewToken() result;got\n%s\nwant\n%s", got, want)
+		}
+	}
+	// token with accountID only
+	f("1", "1")
+	// token with accountID and projecTID
+	f("1:2", "1:2")
 	// multitenant
 	f("multitenant", "multitenant")
 }
@@ -75,4 +93,6 @@ func TestNewTokenFailure(t *testing.T) {
 	f("a:b:c")
 	// many int parts in the token"
 	f("1:2:3")
+	// multitenant
+	f("multitenant")
 }