Repository: VictoriaMetrics/VictoriaMetrics
Pull request #7202 (2024-10-08 15:04:14)

docs: update stream aggregation docs
This PR is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6777. The differences are the following:
* it keeps backward compatibility for links
* it re-structures only original document file
* it adds #common-mistakes section, re-phrased

### Describe Your Changes

Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.

### Checklist

The following checks are **mandatory**:

- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).



Patch:
diff --git a/Makefile b/Makefile
index e3df51dda..ead25d376 100644
--- a/Makefile
+++ b/Makefile
@@ -265,6 +265,14 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
+release-victoria-logs-bundle: \
+	release-victoria-logs \
+	release-vlogscli
+
+publish-victoria-logs-bundle: \
+	publish-victoria-logs \
+	publish-vlogscli
+
 release-victoria-logs:
 	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
 		release-victoria-logs-linux-amd64 \
diff --git a/app/vlogscli/json_prettifier.go b/app/vlogscli/json_prettifier.go
index 0fe4f6d5f..1369155b4 100644
--- a/app/vlogscli/json_prettifier.go
+++ b/app/vlogscli/json_prettifier.go
@@ -17,6 +17,7 @@ const (
 	outputModeJSONMultiline  = outputMode(0)
 	outputModeJSONSingleline = outputMode(1)
 	outputModeLogfmt         = outputMode(2)
+	outputModeCompact        = outputMode(3)
 )
 
 func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
@@ -31,6 +32,8 @@ func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logsto
 		}
 	case outputModeLogfmt:
 		return writeLogfmtObject
+	case outputModeCompact:
+		return writeCompactObject
 	default:
 		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
 	}
@@ -94,8 +97,13 @@ func (jp *jsonPrettifier) prettifyJSONLines() error {
 		if err := jp.formatter(jp.bw, fields); err != nil {
 			return err
 		}
+
+		// Flush bw after every output line in order to show results as soon as they appear.
+		if err := jp.bw.Flush(); err != nil {
+			return err
+		}
 	}
-	return jp.bw.Flush()
+	return nil
 }
 
 func (jp *jsonPrettifier) Close() error {
@@ -161,6 +169,26 @@ func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
 	return err
 }
 
+func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
+	if len(fields) == 1 {
+		// Just write field value as is without name
+		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
+		return err
+	}
+	if len(fields) == 2 && fields[0].Name == "_time" || fields[1].Name == "_time" {
+		// Write _time\tfieldValue as is
+		if fields[0].Name == "_time" {
+			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
+			return err
+		}
+		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
+		return err
+	}
+
+	// Fall back to logfmt
+	return writeLogfmtObject(w, fields)
+}
+
 func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
 	if len(fields) == 0 {
 		fmt.Fprintf(w, "{}\n")
diff --git a/app/vlogscli/main.go b/app/vlogscli/main.go
index 055cea13f..36acb1720 100644
--- a/app/vlogscli/main.go
+++ b/app/vlogscli/main.go
@@ -27,9 +27,13 @@ import (
 
 var (
 	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs")
+		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
+	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
+		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
 	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
 	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
+	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
+	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
 )
 
 const (
@@ -95,9 +99,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			case io.EOF:
 				if s != "" {
 					// This is non-interactive query execution.
-					if err := executeQuery(context.Background(), rl, s, outputMode); err != nil {
-						fmt.Fprintf(rl, "%s\n", err)
-					}
+					executeQuery(context.Background(), rl, s, outputMode)
 				}
 				return
 			case readline.ErrInterrupt:
@@ -147,6 +149,13 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			s = ""
 			continue
 		}
+		if s == `\c` {
+			fmt.Fprintf(rl, "compact output mode\n")
+			outputMode = outputModeCompact
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
 		if s == `\logfmt` {
 			fmt.Fprintf(rl, "logfmt output mode\n")
 			outputMode = outputModeLogfmt
@@ -163,18 +172,9 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 
 		// Execute the query
 		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		err = executeQuery(ctx, rl, s, outputMode)
+		executeQuery(ctx, rl, s, outputMode)
 		cancel()
 
-		if err != nil {
-			if errors.Is(err, context.Canceled) {
-				fmt.Fprintf(rl, "\n")
-			} else {
-				fmt.Fprintf(rl, "%s\n", err)
-			}
-			// Save queries in the history even if they weren't finished successfully
-		}
-
 		historyLines = pushToHistory(rl, historyLines, s)
 		s = ""
 		rl.SetPrompt(firstLinePrompt)
@@ -257,26 +257,90 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
+\c - compact output
 \logfmt - logfmt output mode
+\tail <query> - live tail <query> results
 `)
 }
 
-func executeQuery(ctx context.Context, output io.Writer, s string, outputMode outputMode) error {
+func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
+	if strings.HasPrefix(qStr, `\tail `) {
+		tailQuery(ctx, output, qStr, outputMode)
+		return
+	}
+
+	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
+	if respBody == nil {
+		return
+	}
+	defer func() {
+		_ = respBody.Close()
+	}()
+
+	if err := readWithLess(respBody); err != nil {
+		fmt.Fprintf(output, "error when reading query response: %s\n", err)
+		return
+	}
+}
+
+func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
+	qStr = strings.TrimPrefix(qStr, `\tail `)
+	qURL, err := getTailURL()
+	if err != nil {
+		fmt.Fprintf(output, "%s\n", err)
+		return
+	}
+
+	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
+	if respBody == nil {
+		return
+	}
+	defer func() {
+		_ = respBody.Close()
+	}()
+
+	if _, err := io.Copy(output, respBody); err != nil {
+		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
+			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
+		}
+		fmt.Fprintf(output, "\n")
+		return
+	}
+}
+
+func getTailURL() (string, error) {
+	if *tailURL != "" {
+		return *tailURL, nil
+	}
+
+	u, err := url.Parse(*datasourceURL)
+	if err != nil {
+		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
+	}
+	if !strings.HasSuffix(u.Path, "/query") {
+		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
+	}
+	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
+	return u.String(), nil
+}
+
+func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
 	// Parse the query and convert it to canonical view.
-	s = strings.TrimSuffix(s, ";")
-	q, err := logstorage.ParseQuery(s)
+	qStr = strings.TrimSuffix(qStr, ";")
+	q, err := logstorage.ParseQuery(qStr)
 	if err != nil {
-		return fmt.Errorf("cannot parse query: %w", err)
+		fmt.Fprintf(output, "cannot parse query: %s\n", err)
+		return nil
 	}
-	qStr := q.String()
+	qStr = q.String()
 	fmt.Fprintf(output, "executing [%s]...", qStr)
 
-	// Prepare HTTP request for VictoriaLogs
+	// Prepare HTTP request for qURL
 	args := make(url.Values)
 	args.Set("query", qStr)
 	data := strings.NewReader(args.Encode())
 
-	req, err := http.NewRequestWithContext(ctx, "POST", *datasourceURL, data)
+	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
 	if err != nil {
 		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
 	}
@@ -284,38 +348,37 @@ func executeQuery(ctx context.Context, output io.Writer, s string, outputMode ou
 	for _, h := range headers {
 		req.Header.Set(h.Name, h.Value)
 	}
+	req.Header.Set("AccountID", strconv.Itoa(*accountID))
+	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
 
-	// Execute HTTP request at VictoriaLogs
+	// Execute HTTP request at qURL
 	startTime := time.Now()
 	resp, err := httpClient.Do(req)
 	queryDuration := time.Since(startTime)
 	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
 	if err != nil {
-		return fmt.Errorf("cannot execute query: %w", err)
+		if errors.Is(err, context.Canceled) {
+			fmt.Fprintf(output, "\n")
+		} else {
+			fmt.Fprintf(output, "cannot execute query: %s\n", err)
+		}
+		return nil
 	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
 
+	// Verify response code
 	if resp.StatusCode != http.StatusOK {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
 		}
-		return fmt.Errorf("unexpected status code: %d; response body:\n%s", resp.StatusCode, body)
+		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
+		return nil
 	}
 
-	// Prettify the response and stream it to 'less'.
+	// Prettify the response body
 	jp := newJSONPrettifier(resp.Body, outputMode)
-	defer func() {
-		_ = jp.Close()
-	}()
-
-	if err := readWithLess(jp); err != nil {
-		return fmt.Errorf("error when reading query response: %w", err)
-	}
 
-	return nil
+	return jp
 }
 
 var httpClient = &http.Client{}
diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go
index 716652383..170017c7b 100644
--- a/app/vlselect/logsql/logsql.go
+++ b/app/vlselect/logsql/logsql.go
@@ -394,7 +394,9 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; "+
+			"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+		return
 	}
 	q.Optimize()
 
diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml
index 7d1d86e0b..fe701922f 100644
--- a/deployment/docker/docker-compose-victorialogs.yml
+++ b/deployment/docker/docker-compose-victorialogs.yml
@@ -40,7 +40,7 @@ services:
 #  storing logs and serving read queries.
   victorialogs:
     container_name: victorialogs
-    image: docker.io/victoriametrics/victoria-logs:v0.34.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.35.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
       - "--httpListenAddr=:9428"
diff --git a/deployment/docker/victorialogs/compose-base.yml b/deployment/docker/victorialogs/compose-base.yml
index a5875c135..64985ada5 100644
--- a/deployment/docker/victorialogs/compose-base.yml
+++ b/deployment/docker/victorialogs/compose-base.yml
@@ -1,7 +1,7 @@
 services:
   # meta service will be ignored by compose
   .victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.34.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.35.0-victorialogs
     command:
       - -storageDataPath=/vlogs
       - -loggerFormat=json
diff --git a/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml b/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml
index 58a68e97d..84d09b4ae 100644
--- a/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml
+++ b/deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml
@@ -73,7 +73,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.16.1
+    image: victoriametrics/vmanomaly:v1.16.3
     depends_on:
       - "victoriametrics"
     ports:
diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml
index 83740606c..79d64d4b7 100644
--- a/deployment/logs-benchmark/docker-compose.yml
+++ b/deployment/logs-benchmark/docker-compose.yml
@@ -3,7 +3,7 @@ version: '3'
 services:
   # Run `make package-victoria-logs` to build victoria-logs image
   vlogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.34.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.35.0-victorialogs
     volumes:
       - vlogs:/vlogs
     ports:
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index 6c5d107a3..533f5488e 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -15,6 +15,12 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+## [v0.35.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.35.0-victorialogs)
+
+* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add ability to live tail query results - see [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#live-tailing).
+* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add compact output mode for query results. It can be enabled by typing `\c` and then pressing `enter`. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#output-modes).
+* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add `-accountID` and `-projectID` command-line flags for setting `AccountID` and `ProjectID` values when querying the specific [tenants](https://docs.victoriametrics.com/victorialogs/#multitenancy).
+
 ## [v0.34.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.34.0-victorialogs)
 
 Released at 2024-10-08
diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md
index fa37a3f73..332b009cf 100644
--- a/docs/VictoriaLogs/LogsQL.md
+++ b/docs/VictoriaLogs/LogsQL.md
@@ -36,7 +36,7 @@ For example, the following query finds all the logs with `error` word:
 error
 ```
 
-See [how to send queries to VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
+It is recommended to use [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/) for querying VictoriaLogs.
 
 If the queried [word](#word) clashes with LogsQL keywords, then just wrap it into quotes.
 For example, the following query finds all the log messages with `and` [word](#word):
diff --git a/docs/VictoriaLogs/QuickStart.md b/docs/VictoriaLogs/QuickStart.md
index e6abe8493..d8a2b0be3 100644
--- a/docs/VictoriaLogs/QuickStart.md
+++ b/docs/VictoriaLogs/QuickStart.md
@@ -33,8 +33,8 @@ Just download archive for the needed Operating system and architecture, unpack i
 For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.34.0-victorialogs/victoria-logs-linux-amd64-v0.34.0-victorialogs.tar.gz
-tar xzf victoria-logs-linux-amd64-v0.34.0-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.35.0-victorialogs/victoria-logs-linux-amd64-v0.35.0-victorialogs.tar.gz
+tar xzf victoria-logs-linux-amd64-v0.35.0-victorialogs.tar.gz
 ./victoria-logs-prod
 ```
 
@@ -58,7 +58,7 @@ Here is the command to run VictoriaLogs in a Docker container:
 
 ```sh
 docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
-  docker.io/victoriametrics/victoria-logs:v0.34.0-victorialogs
+  docker.io/victoriametrics/victoria-logs:v0.35.0-victorialogs
 ```
 
 See also:
diff --git a/docs/VictoriaLogs/data-ingestion/Vector.md b/docs/VictoriaLogs/data-ingestion/Vector.md
index 85ac76749..a3ff30f01 100644
--- a/docs/VictoriaLogs/data-ingestion/Vector.md
+++ b/docs/VictoriaLogs/data-ingestion/Vector.md
@@ -48,7 +48,7 @@ for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.co
 sinks:
   vlogs:
     type: "loki"
-    endpoint = "http://localhost:9428/insert/loki/"
+    endpoint: "http://localhost:9428/insert/loki/"
     inputs:
       - your_input
     compression: gzip
diff --git a/docs/VictoriaLogs/querying/README.md b/docs/VictoriaLogs/querying/README.md
index d52f4c3e8..20ef6c700 100644
--- a/docs/VictoriaLogs/querying/README.md
+++ b/docs/VictoriaLogs/querying/README.md
@@ -1,10 +1,10 @@
 [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can be queried with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
 via the following ways:
 
+- [Command-line interface](#command-line)
+- [HTTP API](#http-api)
 - [Web UI](#web-ui) - a web-based UI for querying logs
 - [Grafana plugin](#visualization-in-grafana)
-- [HTTP API](#http-api)
-- [Command-line interface](#command-line)
 
 ## HTTP API
 
@@ -828,7 +828,7 @@ curl http://localhost:9428/select/logsql/query -d 'query=error'
 If the command above returns "never-ending" response, then just press `ctrl+C` at any time in order to cancel the query.
 VictoriaLogs notices that the response stream is closed, so it cancels the query and stops consuming CPU, RAM and disk IO for this query.
 
-Then just use `head` command for investigating the returned log messages and narrowing down the query:
+Then use `head` command for investigating the returned log messages and narrowing down the query:
 
 ```sh
 curl http://localhost:9428/select/logsql/query -d 'query=error' | head -10
@@ -865,7 +865,8 @@ curl http://localhost:9428/select/logsql/query -d 'query=error AND "cannot open
 ```
 
 Note that the `query` arg must be properly encoded with [percent encoding](https://en.wikipedia.org/wiki/URL_encoding) when passing it to `curl`
-or similar tools.
+or similar tools. It is highly recommended to use [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/) -
+it automatically performs all the needed encoding.
 
 The `pipe the query to "head" or "less" -> investigate the results -> refine the query` iteration
 can be repeated multiple times until the needed log messages are found.
diff --git a/docs/VictoriaLogs/querying/vlogscli.md b/docs/VictoriaLogs/querying/vlogscli.md
index 440e5be95..9515d52dd 100644
--- a/docs/VictoriaLogs/querying/vlogscli.md
+++ b/docs/VictoriaLogs/querying/vlogscli.md
@@ -15,17 +15,19 @@ or from [docker images](https://hub.docker.com/r/victoriametrics/vlogscli/tags):
 ### Running `vlogscli` from release binary
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.34.0-victorialogs/vlogscli-linux-amd64-v0.34.0-victorialogs.tar.gz
-tar xzf vlogscli-linux-amd64-v0.34.0-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.35.0-victorialogs/vlogscli-linux-amd64-v0.35.0-victorialogs.tar.gz
+tar xzf vlogscli-linux-amd64-v0.35.0-victorialogs.tar.gz
 ./vlogscli-prod
 ```
 
 ### Running `vlogscli` from Docker image
 
 ```sh
-docker run --rm -it docker.io/victoriametrics/vlogscli:v0.34.0-victorialogs
+docker run --rm -it docker.io/victoriametrics/vlogscli:v0.35.0-victorialogs
 ```
 
+## Configuration
+
 By default `vlogscli` sends queries to [`http://localhost:8429/select/logsql/query`](https://docs.victoriametrics.com/victorialogs/querying/#querying-logs).
 The url to query can be changed via `-datasource.url` command-line flag. For example, the following command instructs
 `vlogsql` sending queries to `https://victoria-logs.some-domain.com/select/logsql/query`:
@@ -42,6 +44,14 @@ which queries `(AccountID=123, ProjectID=456)` [tenant](https://docs.victoriamet
 ./vlogsql -header='AccountID: 123' -header='ProjectID: 456'
 ```
 
+`AccountID` and `ProjectID` values can be also set via `-accountID` and `-projectID` command-line flags:
+
+```sh
+./vlogsql -accountID=123 -projectID=456
+```
+
+## Querying
+
 After the start `vlogsql` provides a prompt for writing [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries.
 The query can be multi-line. It is sent to VictoriaLogs as soon as it contains `;` at the end or if a blank line follows the query.
 For example:
@@ -77,6 +87,17 @@ See also [`less` docs](https://man7.org/linux/man-pages/man1/less.1.html) and
 [command-line integration docs for VictoriaMetrics](https://docs.victoriametrics.com/victorialogs/querying/#command-line).
 
 
+## Live tailing
+
+`vlogsql` enters live tailing mode when the query is prepended with `\tail ` command. For example:
+
+```
+;> \tail {kubernetes_container_name="vmagent"};
+```
+
+By default `vlogscli` derives [the URL for live tailing](https://docs.victoriametrics.com/victorialogs/querying/#live-tailing) from the `-datasource.url` command-line flag
+by replacing `/query` with `/tail` at the end of `-datasource.url`. The URL for live tailing can be specified explicitly via `-tail.url` command-line flag.
+
 ## Query history
 
 `vlogsql` supports query history - press `up` and `down` keys for navigating the history.
@@ -90,6 +111,7 @@ Press `Enter` when the needed query is found in order to execute it.
 Press `Ctrl+C` for exit from the `search history` mode.
 See also [other available shortcuts](https://github.com/chzyer/readline/blob/f533ef1caae91a1fcc90875ff9a5a030f0237c6a/doc/shortcut.md).
 
+
 ## Output modes
 
 By default `vlogscli` displays query results as prettified JSON object with every field on a separate line.
@@ -99,4 +121,8 @@ Fields in every JSON object are sorted in alphabetical order. This simplifies lo
 
 * A single JSON line per every result. Type `\s` and press `enter` for this mode.
 * Multline JSON per every result. Type `\m` and press `enter` for this mode.
+* Compact output. Type `\c` and press `enter` for this mode.
+  This mode shows field values as is if the response contains a single field
+  (for example if [`fields _msg` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#fields-pipe) is used)
+  plus optional [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field).
 * [Logfmt output](https://brandur.org/logfmt). Type `\logfmt` and press `enter` for this mode.
diff --git a/docs/anomaly-detection/CHANGELOG.md b/docs/anomaly-detection/CHANGELOG.md
index 85c5bfe5e..f6e4d60dc 100644
--- a/docs/anomaly-detection/CHANGELOG.md
+++ b/docs/anomaly-detection/CHANGELOG.md
@@ -11,6 +11,10 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.16.3
+Released: 2024-10-08
+- IMPROVEMENT: Added `tls_cert_file` and `tls_key_file` arguments to support mTLS (mutual TLS) in `vmanomaly` components. This enhancement applies to the following components: [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). You can also use these arguments in conjunction with `verify_tls` when it is set as a path to a custom CA certificate file.
+
 ## v1.16.2
 Released: 2024-10-06
 - FEATURE: Added support for `multitenant` value in `tenant_id` arg to enable querying across multiple tenants in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) (option available from [v1.104.0](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels)):
@@ -72,7 +76,7 @@ Released: 2024-08-14
 ## v1.15.2
 Released: 2024-08-13
 - IMPROVEMENT: Enhanced [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) (e.g., [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)) to automatically create model instances for unseen time series during `infer` calls, eliminating the need to wait for the next `fit` call. This ensures no inferences are skipped **when using online models**.
-- FIX: Corrected an issue with the [`OnlineMADModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) to ensure proper functionality when used in combination with [on-disk model dump mode](https://docs.victoriametrics.com/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
+- FIX: Corrected an issue with the [`OnlineMADModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) to ensure proper functionality when used in combination with [on-disk model dump mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
 - FIX: Addressed numerical instability in the [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) when `use_transform` is set to `True`.
 - FIX: Resolved a logging issue that could cause a `RuntimeError: reentrant call inside <_io.BufferedWriter name='<stderr>'>` when a termination event was received.
 
@@ -120,7 +124,7 @@ Released: 2024-07-17
 Released: 2024-07-15
 - IMPROVEMENT: update `node-exporter` [preset](https://docs.victoriametrics.com/anomaly-detection/presets/#node-exporter) to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)
 - FIX: add `verify_tls` arg for [`push`](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters) monitoring section. Also, `verify_tls` is now correctly used in [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer).
-- FIX: now [`AutoTuned`](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned) model wrapper works correctly in [on-disk model storage mode](https://docs.victoriametrics.com/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
+- FIX: now [`AutoTuned`](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned) model wrapper works correctly in [on-disk model storage mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
 - FIX: now [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models), like [`RollingQuantile`](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) are properly handled in [One-off scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#oneoff-scheduler), when wrapped in [`AutoTuned`](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned)
 
 ## v1.13.0
diff --git a/docs/anomaly-detection/FAQ.md b/docs/anomaly-detection/FAQ.md
index 45dc9c8dc..773ead709 100644
--- a/docs/anomaly-detection/FAQ.md
+++ b/docs/anomaly-detection/FAQ.md
@@ -118,7 +118,9 @@ writer:
 Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_frequency]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](https://docs.victoriametrics.com/anomaly-detection/components/scheduler#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](https://docs.victoriametrics.com/anomaly-detection/components/writer#vm-writer) for further visualization (i.e. in VMUI or Grafana)
 
 ## Resource consumption of vmanomaly
-`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models). Its resource usage is directly related to these factors, making it adaptable to various operational scales.
+`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.
+
+### On-disk mode
 
 > **Note**: Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130), there is an option to save anomaly detection models to the host filesystem after the `fit` stage (instead of keeping them in memory by default). This is particularly useful for **resource-intensive setups** (e.g., many models, many metrics, or larger [`fit_window` argument](https://docs.victoriametrics.com/anomaly-detection/components/scheduler#periodic-scheduler-config-example)) and for 3rd-party models that store fit data (such as [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models#prophet) or [HoltWinters](https://docs.victoriametrics.com/anomaly-detection/components/models#holt-winters)). This reduces RAM consumption significantly, though at the cost of slightly slower `infer` stages. To enable this, set the environment variable `VMANOMALY_MODEL_DUMPS_DIR` to the desired location. If using [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md), starting from chart version `1.3.0` `.persistentVolume.enabled` should be set to `true` in [values.yaml](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/values.yaml).
 
@@ -130,7 +132,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.16.1
+    image: victoriametrics/vmanomaly:v1.16.3
     # ...
     ports:
       - "8490:8490"
@@ -156,9 +158,11 @@ volumes:
   vmanomaly_data_dump_dir: {}
 ```
 
-> **Note**: Starting from [v1.15.0](https://docs.victoriametrics.com/anomaly-detection/changelog#v1150) with the introduction of [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models), you can additionally reduce resource consumption (e.g., flatten `fit` stage peaks by querying less data from VictoriaMetrics at once).
+For Helm chart users, refer to the `persistentVolume` [section](https://github.com/VictoriaMetrics/helm-charts/blob/7f5a2c00b14c2c088d7d8d8bcee7a440a5ff11c6/charts/victoria-metrics-anomaly/values.yaml#L183) in the [`values.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/values.yaml) file. Ensure that the boolean flags `dumpModels` and `dumpData` are set as needed (both are *enabled* by default).
 
-**Additional Benefits of Switching to Online Models**:
+### Online models
+
+> **Note**: Starting from [v1.15.0](https://docs.victoriametrics.com/anomaly-detection/changelog#v1150) with the introduction of [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models), you can additionally reduce resource consumption (e.g., flatten `fit` stage peaks by querying less data from VictoriaMetrics at once).
 
 - **Reduced Latency**: Online models update incrementally, which can lead to faster response times for anomaly detection since the model continuously adapts to new data without waiting for a batch `fit`.
 - **Scalability**: Handling smaller data chunks at a time reduces memory and computational overhead, making it easier to scale the anomaly detection system.
diff --git a/docs/anomaly-detection/Overview.md b/docs/anomaly-detection/Overview.md
index 2d6b33581..0db99ecfe 100644
--- a/docs/anomaly-detection/Overview.md
+++ b/docs/anomaly-detection/Overview.md
@@ -229,7 +229,7 @@ This will expose metrics at `http://0.0.0.0:8080/metrics` page.
 To use *vmanomaly* you need to pull docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.16.1
+docker pull victoriametrics/vmanomaly:v1.16.3
 ```
 
 > Note: please check what is latest release in [CHANGELOG](https://docs.victoriametrics.com/anomaly-detection/changelog/)
@@ -239,7 +239,7 @@ docker pull victoriametrics/vmanomaly:v1.16.1
 You can put a tag on it for your convenience:
 
 ```sh
-docker image tag victoriametrics/vmanomaly:v1.16.1 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.16.3 vmanomaly
 ```
 Here is an example of how to run *vmanomaly* docker container with [license file](#licensing):
 
diff --git a/docs/anomaly-detection/QuickStart.md b/docs/anomaly-detection/QuickStart.md
index a6d59519f..2252eaf6a 100644
--- a/docs/anomaly-detection/QuickStart.md
+++ b/docs/anomaly-detection/QuickStart.md
@@ -21,7 +21,9 @@ The following options are available:
 - [To run Docker image](#docker)
 - [To run in Kubernetes with Helm charts](#kubernetes-with-helm-charts)
 
-> **Note**: Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130) there is a mode to keep anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default); This may lead to **noticeable reduction of RAM used** on bigger setups. See instructions [here](https://docs.victoriametrics.com/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
+> **Note**: Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130) there is a mode to keep anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default); This may lead to **noticeable reduction of RAM used** on bigger setups. See instructions [here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
+
+> **Note**: Starting from [v1.16.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1160), a similar optimization is available for data read from VictoriaMetrics TSDB. See instructions [here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
 
 ### Docker
 
@@ -32,13 +34,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.16.1
+docker pull victoriametrics/vmanomaly:v1.16.3
 ```
 
 2. (Optional step) tag the `vmanomaly` Docker image:
 
 ```sh
-docker image tag victoriametrics/vmanomaly:v1.16.1 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.16.3 vmanomaly
 ```
 
 3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -70,7 +72,7 @@ docker run -it --user 1000:1000 \
 services:
   # ...
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.16.1
+    image: victoriametrics/vmanomaly:v1.16.3
     volumes:
         $YOUR_LICENSE_FILE_PATH:/license
         $YOUR_CONFIG_FILE_PATH:/config.yml
diff --git a/docs/anomaly-detection/components/models.md b/docs/anomaly-detection/components/models.md
index 460fbfedb..4ebd669c1 100644
--- a/docs/anomaly-detection/components/models.md
+++ b/docs/anomaly-detection/components/models.md
@@ -17,7 +17,7 @@ This section covers the `Models` component of VictoriaMetrics Anomaly Detection
 - You can also integrate a **custom model**—see the [custom model guide](#custom-model-guide) for more details.
 - Models have **different types and properties**—refer to the [model types section](#model-types) for more information.
 
-> **Note:** Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130), models can be dumped to disk instead of being stored in RAM. This option **slightly reduces inference speed but significantly decreases RAM usage**, particularly useful for larger setups. For more details, see the [relevant FAQ section](https://docs.victoriametrics.com/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
+> **Note:** Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130), models can be dumped to disk instead of being stored in RAM. This option **slightly reduces inference speed but significantly decreases RAM usage**, particularly useful for larger setups. For more details, see the [relevant FAQ section](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
 
 > **Note:** Starting from [v1.10.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1100) model section in config supports multiple models via aliasing. <br>Also, `vmanomaly` expects model section to be named `models`. Using old (flat) format with `model` key is deprecated and will be removed in future versions. Having `model` and `models` sections simultaneously in a config will result in only `models` being used:
 
@@ -366,6 +366,7 @@ Infer stage
 - The ability to distribute the data load evenly between the initial `fit` and subsequent `infer` calls. For example, an online model can be fit on 10 `1m` datapoints during the initial `fit` stage once per month and then be gradually updated on the same 10 `1m` datapoints during each `infer` call each 10 minutes.
 - The model can adapt to new data patterns (gradually updating itself during each `infer` call) without needing to wait for the next `fit` call and one big re-training.
 - Slightly faster training/updating times compared to similar offline models.
+- Please refer to additional benefits for data-intensive setups in correspondent [FAQ](https://docs.victoriametrics.com/anomaly-detection/faq#online-models) section.
 
 **Limitations**:
 
@@ -446,7 +447,7 @@ models:
 > **Note**: There are some expected limitations of Autotune mode:
 > - It can't be made on your [custom model](#custom-model-guide).
 > - It can't be applied to itself (like `tuned_class_name: 'model.auto.AutoTunedModel'`)
-> - `AutoTunedModel` can't be used on [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) like [`RollingQuantile`](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) in combination with [on-disk model storage mode](https://docs.victoriametrics.com/anomaly-detection/faq/#resource-consumption-of-vmanomaly), as the rolling models exists only during `infer` calls and aren't persisted neither in RAM, nor on disk.
+> - `AutoTunedModel` can't be used on [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) like [`RollingQuantile`](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) in combination with [on-disk model storage mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode), as the rolling models exists only during `infer` calls and aren't persisted neither in RAM, nor on disk.
 
 
 ### [Prophet](https://facebook.github.io/prophet/)
@@ -961,7 +962,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.16.1
+docker pull victoriametrics/vmanomaly:v1.16.3
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -975,7 +976,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.16.1 /config.yaml \
+victoriametrics/vmanomaly:v1.16.3 /config.yaml \
 --licenseFile=/license
 ```
 
diff --git a/docs/anomaly-detection/components/monitoring.md b/docs/anomaly-detection/components/monitoring.md
index 87adcdb90..a2563b350 100644
--- a/docs/anomaly-detection/components/monitoring.md
+++ b/docs/anomaly-detection/components/monitoring.md
@@ -134,14 +134,38 @@ Path to a file, which contains token, that is passed in the standard format with
         </tr>
         <tr>
             <td>
-
 `verify_tls`
             </td>
             <td>
-
-`False`
+`false`
+            </td>
+            <td>
+Verify TLS certificate. If `False`, it will not verify the TLS certificate. 
+If `True`, it will verify the certificate using the system's CA store. 
+If a path to a CA bundle file (like `ca.crt`), it will verify the certificate using the provided CA bundle.
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_cert_file`
+            </td>
+            <td>
+`path/to/cert.crt`
+            </td>
+            <td>
+Path to a file with the client certificate, i.e. `client.crt`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_key_file`
+            </td>
+            <td>
+`path/to/key.crt`
+            </td>
+            <td>
+Path to a file with the client certificate key, i.e. `client.key`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
             </td>
-            <td>Allows disabling TLS verification of the remote certificate.</td>
         </tr>
         <tr>
             <td>
@@ -184,6 +208,12 @@ monitoring:
       test: "test-1"
 ```
 
+## mTLS protection
+
+Starting from [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163), `vmanomaly` components such as [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) support [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) to ensure secure communication with [VictoriaMetrics Enterprise, configured with mTLS](https://docs.victoriametrics.com/#mtls-protection).
+
+For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_cert_file`, and `tls_key_file`, please refer to the [mTLS protection section](https://docs.victoriametrics.com/anomaly-detection/components/reader/#mtls-protection) in the [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) documentation. The configuration principles apply consistently across all these `vmanomaly` components.
+
 ## Metrics generated by vmanomaly
 
 <table class="params">
diff --git a/docs/anomaly-detection/components/reader.md b/docs/anomaly-detection/components/reader.md
index 213ec0a52..8ef3b16f3 100644
--- a/docs/anomaly-detection/components/reader.md
+++ b/docs/anomaly-detection/components/reader.md
@@ -211,7 +211,31 @@ Timeout for the requests, passed as a string
 `false`
             </td>
             <td>
-Allows disabling TLS verification of the remote certificate.
+Verify TLS certificate. If `False`, it will not verify the TLS certificate. 
+If `True`, it will verify the certificate using the system's CA store. 
+If a path to a CA bundle file (like `ca.crt`), it will verify the certificate using the provided CA bundle.
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_cert_file`
+            </td>
+            <td>
+`path/to/cert.crt`
+            </td>
+            <td>
+Path to a file with the client certificate, i.e. `client.crt`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_key_file`
+            </td>
+            <td>
+`path/to/key.crt`
+            </td>
+            <td>
+Path to a file with the client certificate key, i.e. `client.key`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
             </td>
         </tr>
         <tr>
@@ -289,6 +313,50 @@ reader:
   latency_offset: '1ms'
 ```
 
+<!-- ### mTLS protection
+
+Starting from [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163), `vmanomaly` supports [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) requests in its components, like [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters) to query from and write to [VictoriaMetrics Enterprise, configured in the same mode](https://docs.victoriametrics.com/#mtls-protection).
+
+Please see the description of next arguments in a [config](#config-parameters):
+- `verify_tls` (if string, acts similar to `-mtlsCAFile` command line arg of VictoriaMetrics).
+- `tls_cert_file` (if given, acts similar to `-tlsCertFile` command line arg of VictoriaMetrics).
+- `tls_key_file` (if given, acts similar to `-tlsKeyFile` command line arg of VictoriaMetrics). -->
+
+
+### mTLS protection
+
+As of [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163), `vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication) for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/#mtls-protection).
+
+mTLS ensures that both the client and server verify each other's identity using certificates, which enhances security by preventing unauthorized access. 
+
+To configure mTLS, the following parameters can be set in the [config](#config-parameters):
+- `verify_tls`: If set to a string, it functions like the `-mtlsCAFile` command-line argument of VictoriaMetrics, specifying the CA bundle to use. Set to `True` to use the system's default certificate store.
+- `tls_cert_file`: Specifies the path to the client certificate, analogous to the `-tlsCertFile` argument of VictoriaMetrics.
+- `tls_key_file`: Specifies the path to the client certificate key, similar to the `-tlsKeyFile` argument of VictoriaMetrics.
+
+These options allow you to securely interact with mTLS-enabled VictoriaMetrics endpoints.
+
+Example configuration to enable mTLS with custom certificates:
+
+```yaml
+reader:
+  class: "vm"
+  datasource_url: "https://your-victoriametrics-instance-with-mtls"
+  # tenant_id: "0:0" uncomment and set for cluster version
+  queries:
+    vm_blocks_example:
+      expr: 'avg(rate(vm_blocks[5m]))'
+      step: 30s
+  sampling_period: 30s
+  verify_tls: "path/to/ca.crt"  # path to CA bundle for TLS verification
+  tls_cert_file: "path/to/client.crt"  # path to the client certificate
+  tls_key_file:  "path/to/client.key"  # path to the client certificate key
+  # additional reader parameters ...
+
+# other config sections, like models, schedulers, writer, ...
+```
+
+
 ### Healthcheck metrics
 
 `VmReader` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
diff --git a/docs/anomaly-detection/components/writer.md b/docs/anomaly-detection/components/writer.md
index 343ce30e6..a72e8cd65 100644
--- a/docs/anomaly-detection/components/writer.md
+++ b/docs/anomaly-detection/components/writer.md
@@ -185,16 +185,37 @@ Timeout for the requests, passed as a string
         </tr>
         <tr>
             <td>
-
 `verify_tls`
             </td>
             <td>
-
 `false`
             </td>
             <td>
-
-Allows disabling TLS verification of the remote certificate.
+Verify TLS certificate. If `False`, it will not verify the TLS certificate. 
+If `True`, it will verify the certificate using the system's CA store. 
+If a path to a CA bundle file (like `ca.crt`), it will verify the certificate using the provided CA bundle.
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_cert_file`
+            </td>
+            <td>
+`path/to/cert.crt`
+            </td>
+            <td>
+Path to a file with the client certificate, i.e. `client.crt`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
+            </td>
+        </tr>
+        <tr>
+            <td>
+`tls_key_file`
+            </td>
+            <td>
+`path/to/key.crt`
+            </td>
+            <td>
+Path to a file with the client certificate key, i.e. `client.key`. Available since [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163).
             </td>
         </tr>
         <tr>
@@ -272,6 +293,13 @@ Please note the different behaviors depending on the `tenant_id` value:
     or ensure that reserved key `vm_account_id` is not explicitly set for single-tenant environments.
     ```
 
+### mTLS protection
+
+Starting from [v1.16.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1163), `vmanomaly` components such as [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) support [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) to ensure secure communication with [VictoriaMetrics Enterprise, configured with mTLS](https://docs.victoriametrics.com/#mtls-protection).
+
+For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_cert_file`, and `tls_key_file`, please refer to the [mTLS protection section](https://docs.victoriametrics.com/anomaly-detection/components/reader/#mtls-protection) in the [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) documentation. The configuration principles apply consistently across all these `vmanomaly` components.
+
+
 ### Healthcheck metrics
 
 `VmWriter` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics). 
diff --git a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
index e76008c2c..5df96ed2a 100644
--- a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
+++ b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
@@ -385,7 +385,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.16.1
+    image: victoriametrics/vmanomaly:v1.16.3
     depends_on:
       - "victoriametrics"
     ports:
diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index f795bd62f..bac335bd5 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -18,147 +18,7 @@ after applying all the configured [relabeling stages](https://docs.victoriametri
 **By default, stream aggregation ignores timestamps associated with the input [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples).
 It expects that the ingested samples have timestamps close to the current time. See [how to ignore old samples](#ignoring-old-samples).**
 
-## Configuration
-
-Stream aggregation can be configured via the following command-line flags:
-
-- `-streamAggr.config` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/)
-  and at [vmagent](https://docs.victoriametrics.com/vmagent/).
-- `-remoteWrite.streamAggr.config` at [vmagent](https://docs.victoriametrics.com/vmagent/) only. This flag can be specified individually
-  per each `-remoteWrite.url`, so the aggregation happens independently per each remote storage destination.
-  This allows writing different aggregates to different remote storage systems.
-
-These flags must point to a file containing [stream aggregation config](#stream-aggregation-config).
-The file may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
-
-By default, the following data is written to the storage when stream aggregation is enabled:
-
-- the aggregated samples;
-- the raw input samples, which didn't match any `match` option in the provided [config](#stream-aggregation-config).
-
-This behaviour can be changed via the following command-line flags:
-
-- `-streamAggr.keepInput` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) 
-  and [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
-  `-remoteWrite.streamAggr.keepInput` flag can be specified individually per each `-remoteWrite.url`.
-  If one of these flags is set, then all the input samples are written to the storage alongside the aggregated samples.
-- `-streamAggr.dropInput` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) 
-  and [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
-  `-remoteWrite.streamAggr.dropInput` flag can be specified individually per each `-remoteWrite.url`.
-  If one of these flags are set, then all the input samples are dropped, while only the aggregated samples are written to the storage.
-
-## Routing
-
-[Single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) supports relabeling,
-deduplication and stream aggregation for all the received data, scraped or pushed. 
-The processed data is then stored in local storage and **can't be forwarded further**.
-
-[vmagent](https://docs.victoriametrics.com/vmagent/) supports relabeling, deduplication and stream aggregation for all 
-the received data, scraped or pushed. Then, the collected data will be forwarded to specified `-remoteWrite.url` destinations.
-The data processing order is the following:
-
-1. all the received data is relabeled according to the specified [`-remoteWrite.relabelConfig`](https://docs.victoriametrics.com/vmagent/#relabeling) (if it is set)
-1. all the received data is deduplicated according to specified [`-streamAggr.dedupInterval`](https://docs.victoriametrics.com/stream-aggregation/#deduplication)
-   (if it is set to duration bigger than 0)
-1. all the received data is aggregated according to specified [`-streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration) (if it is set)
-1. the resulting data is then replicated to each `-remoteWrite.url`
-1. data sent to each `-remoteWrite.url` can be additionally relabeled according to the corresponding `-remoteWrite.urlRelabelConfig` (set individually per URL)
-1. data sent to each `-remoteWrite.url` can be additionally deduplicated according to the corresponding `-remoteWrite.streamAggr.dedupInterval` (set individually per URL)
-1. data sent to each `-remoteWrite.url` can be additionally aggregated according to the corresponding `-remoteWrite.streamAggr.config` (set individually per URL)
-   It isn't recommended using `-streamAggr.config` and `-remoteWrite.streamAggr.config` simultaneously, unless you understand the complications.
-
-Typical scenarios for data routing with `vmagent`:
-
-1. **Aggregate incoming data and replicate to N destinations**. Specify [`-streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration) command-line flag
-   to aggregate the incoming data before replicating it to all the configured `-remoteWrite.url` destinations.
-2. **Individually aggregate incoming data for each destination**. Specify [`-remoteWrite.streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration)
-   command-line flag for each `-remoteWrite.url` destination. [Relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) via `-remoteWrite.urlRelabelConfig`
-   can be used for routing only the selected metrics to each `-remoteWrite.url` destination.
-
-## Deduplication
-
-[vmagent](https://docs.victoriametrics.com/vmagent/) supports online [de-duplication](https://docs.victoriametrics.com/#deduplication) of samples
-before sending them to the configured `-remoteWrite.url`. The de-duplication can be enabled via the following options:
-
-- By specifying the desired de-duplication interval via `-streamAggr.dedupInterval` command-line flag for all received data 
-  or via `-remoteWrite.streamAggr.dedupInterval` command-line flag for the particular `-remoteWrite.url` destination.
-  For example, `./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -remoteWrite.streamAggr.dedupInterval=30s` instructs `vmagent` to leave
-  only the last sample per each seen [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) per every 30 seconds.
-  The de-deduplication is performed after applying [relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) and
-  before performing the aggregation.
-
-- By specifying `dedup_interval` option individually per each [stream aggregation config](#stream-aggregation-config) 
-  in `-remoteWrite.streamAggr.config` or `-streamAggr.config` configs.
-
-[Single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) supports two types of de-duplication:
-- After storing the duplicate samples to local storage. See [`-dedup.minScrapeInterval`](https://docs.victoriametrics.com/#deduplication) command-line option.
-- Before storing the duplicate samples to local storage. This type of de-duplication can be enabled via the following options:
-  - By specifying the desired de-duplication interval via `-streamAggr.dedupInterval` command-line flag.
-    For example, `./victoria-metrics -streamAggr.dedupInterval=30s` instructs VictoriaMetrics to leave only the last sample per each
-    seen [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) per every 30 seconds.
-    The de-duplication is performed after applying `-relabelConfig` [relabeling](https://docs.victoriametrics.com/#relabeling).
-
-  - By specifying `dedup_interval` option individually per each [stream aggregation config](#stream-aggregation-config) at `-streamAggr.config`.
-
-It is possible to drop the given labels before applying the de-duplication. See [these docs](#dropping-unneeded-labels).
-
-The online de-duplication uses the same logic as [`-dedup.minScrapeInterval` command-line flag](https://docs.victoriametrics.com/#deduplication) at VictoriaMetrics.
-
-## Ignoring old samples
-
-By default, all the input samples are taken into account during stream aggregation. If samples with old timestamps 
-outside the current [aggregation interval](#stream-aggregation-config) must be ignored, then the following options can be used:
-
-- To pass `-streamAggr.ignoreOldSamples` command-line flag to [single-node VictoriaMetrics](https://docs.victoriametrics.com/)
-  or to [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
-  `-remoteWrite.streamAggr.ignoreOldSamples` flag can be specified individually per each `-remoteWrite.url`.
-  This enables ignoring old samples for all the [aggregation configs](#stream-aggregation-config).
-
-- To set `ignore_old_samples: true` option at the particular [aggregation config](#stream-aggregation-config).
-  This enables ignoring old samples for that particular aggregation config.
-
-## Ignore aggregation intervals on start
-
-Streaming aggregation results may be incorrect for some time after the restart of [vmagent](https://docs.victoriametrics.com/vmagent/)
-or [single-node VictoriaMetrics](https://docs.victoriametrics.com/) until all the buffered [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
-are sent from remote sources to the `vmagent` or single-node VictoriaMetrics via [supported data ingestion protocols](https://docs.victoriametrics.com/vmagent/#how-to-push-data-to-vmagent).
-In this case it may be a good idea to drop the aggregated data during the first `N` [aggregation intervals](#stream-aggregation-config)
-just after the restart of `vmagent` or single-node VictoriaMetrics. This can be done via the following options:
-
-- The `-streamAggr.ignoreFirstIntervals=N` command-line flag at `vmagent` and single-node VictoriaMetrics. This flag instructs skipping the first `N`
-  [aggregation intervals](#stream-aggregation-config) just after the restart across all the [configured stream aggregation configs](#configuration).
-
-  The `-remoteWrite.streamAggr.ignoreFirstIntervals` command-line flag can be specified individually per each `-remoteWrite.url` at [vmagent](https://docs.victoriametrics.com/vmagent/).
-
-- The `ignore_first_intervals: N` option at the particular [aggregation config](#stream-aggregation-config).
-
-See also:
-
-- [Flush time alignment](#flush-time-alignment)
-- [Ignoring old samples](#ignoring-old-samples)
-
-## Flush time alignment
-
-By default, the time for aggregated data flush is aligned by the `interval` option specified in [aggregate config](#stream-aggregation-config).
-
-For example:
-
-- if `interval: 1m` is set, then the aggregated data is flushed to the storage at the end of every minute
-- if `interval: 1h` is set, then the aggregated data is flushed to the storage at the end of every hour
-
-If you do not need such an alignment, then set `no_align_flush_to_interval: true` option in the [aggregate config](#stream-aggregation-config).
-In this case aggregated data flushes will be aligned to the `vmagent` start time or to [config reload](#configuration-update) time.
-
-The aggregated data on the first and the last interval is dropped during `vmagent` start, restart or [config reload](#configuration-update),
-since the first and the last aggregation intervals are incomplete, so they usually contain incomplete confusing data.
-If you need preserving the aggregated data on these intervals, then set `flush_on_shutdown: true` option in the [aggregate config](#stream-aggregation-config).
-
-See also:
-
-- [Ignore aggregation intervals on start](#ignore-aggregation-intervals-on-start)
-- [Ignoring old samples](#ignoring-old-samples)
-
-## Use cases
+# Use cases
 
 Stream aggregation can be used in the following cases:
 
@@ -167,7 +27,7 @@ Stream aggregation can be used in the following cases:
 * [Reducing the number of stored samples](#reducing-the-number-of-stored-samples)
 * [Reducing the number of stored series](#reducing-the-number-of-stored-series)
 
-### Statsd alternative
+## Statsd alternative
 
 Stream aggregation can be used as [statsd](https://github.com/statsd/statsd) alternative in the following cases:
 
@@ -180,7 +40,7 @@ Stream aggregation can be used as [statsd](https://github.com/statsd/statsd) alt
 Currently, streaming aggregation is available only for [supported data ingestion protocols](https://docs.victoriametrics.com/#how-to-import-time-series-data)
 and not available for [Statsd metrics format](https://github.com/statsd/statsd/blob/master/docs/metric_types.md).
 
-### Recording rules alternative
+## Recording rules alternative
 
 Sometimes [alerting queries](https://docs.victoriametrics.com/vmalert/#alerting-rules) may require non-trivial amounts of CPU, RAM,
 disk IO and network bandwidth at metrics storage side. For example, if `http_request_duration_seconds` histogram is generated by thousands
@@ -212,8 +72,7 @@ See also [aggregating by labels](#aggregating-by-labels).
 
 Field `interval` is recommended to be set to a value at least several times higher than your metrics collect interval.
 
-
-### Reducing the number of stored samples
+## Reducing the number of stored samples
 
 If per-[series](https://docs.victoriametrics.com/keyconcepts/#time-series) samples are ingested at high frequency,
 then this may result in high disk space usage, since too much data must be stored to disk. This also may result
@@ -254,7 +113,7 @@ some_metric:5m_max
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [aggregating histograms](#aggregating-histograms) and [aggregating by labels](#aggregating-by-labels).
 
-### Reducing the number of stored series
+## Reducing the number of stored series
 
 Sometimes applications may generate too many [time series](https://docs.victoriametrics.com/keyconcepts/#time-series).
 For example, the `http_requests_total` metric may have `path` or `user` label with too big number of unique values.
@@ -279,7 +138,7 @@ http_requests_total:30s_without_path_user_total
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [aggregating histograms](#aggregating-histograms).
 
-### Counting input samples
+## Counting input samples
 
 If the monitored application generates event-based metrics, then it may be useful to count the number of such metrics
 at stream aggregation level.
@@ -305,8 +164,7 @@ clicks:30s_count_samples count2
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [aggregating by labels](#aggregating-by-labels).
 
-
-### Summing input metrics
+## Summing input metrics
 
 If the monitored application calculates some events and then sends the calculated number of events to VictoriaMetrics
 at irregular intervals or at too high frequency, then stream aggregation can be used for summing such events
@@ -332,8 +190,7 @@ clicks:1m_sum_samples sum2
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [aggregating by labels](#aggregating-by-labels).
 
-
-### Quantiles over input metrics
+## Quantiles over input metrics
 
 If the monitored application generates measurement metrics per each request, then it may be useful to calculate
 the pre-defined set of [percentiles](https://en.wikipedia.org/wiki/Percentile) over these measurements.
@@ -363,7 +220,7 @@ response_size_bytes:30s_quantiles{quantile="0.99"} value2
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [histograms over input metrics](#histograms-over-input-metrics) and [aggregating by labels](#aggregating-by-labels).
 
-### Histograms over input metrics
+## Histograms over input metrics
 
 If the monitored application generates measurement metrics per each request, then it may be useful to calculate
 a [histogram](https://docs.victoriametrics.com/keyconcepts/#histogram) over these metrics.
@@ -424,10 +281,10 @@ The resulting histogram buckets can be queried with [MetricsQL](https://docs.vic
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [quantiles over input metrics](#quantiles-over-input-metrics) and [aggregating by labels](#aggregating-by-labels).
 
-### Aggregating histograms
+## Aggregating histograms
 
 [Histogram](https://docs.victoriametrics.com/keyconcepts/#histogram) is a set of [counter](https://docs.victoriametrics.com/keyconcepts/#counter)
-metrics with different `vmrange` or `le` labels. As they're counters, the applicable aggregation output is 
+metrics with different `vmrange` or `le` labels. As they're counters, the applicable aggregation output is
 [total](https://docs.victoriametrics.com/stream-aggregation/#total):
 
 ```yaml
@@ -455,88 +312,41 @@ function:
 histogram_quantile(0.9, sum(rate(http_request_duration_seconds_bucket:1m_without_instance_total[5m])) by(le))
 ```
 
-Please note, histograms can be aggregated if their `le` labels are configured identically. 
+Please note, histograms can be aggregated if their `le` labels are configured identically.
 [VictoriaMetrics histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
 have no such requirement.
 
 See [the list of aggregate output](#aggregation-outputs), which can be specified at `output` field.
 See also [histograms over input metrics](#histograms-over-input-metrics) and [quantiles over input metrics](#quantiles-over-input-metrics).
 
-## Output metric names
-
-Output metric names for stream aggregation are constructed according to the following pattern:
-
-```text
-<metric_name>:<interval>[_by_<by_labels>][_without_<without_labels>]_<output>
-```
-
-- `<metric_name>` is the original metric name.
-- `<interval>` is the interval specified in the [stream aggregation config](#stream-aggregation-config).
-- `<by_labels>` is `_`-delimited sorted list of `by` labels specified in the [stream aggregation config](#stream-aggregation-config).
-  If the `by` list is missing in the config, then the `_by_<by_labels>` part isn't included in the output metric name.
-- `<without_labels>` is an optional `_`-delimited sorted list of `without` labels specified in the [stream aggregation config](#stream-aggregation-config).
-  If the `without` list is missing in the config, then the `_without_<without_labels>` part isn't included in the output metric name.
-- `<output>` is the aggregate used for constructing the output metric. The aggregate name is taken from the `outputs` list
-  at the corresponding [stream aggregation config](#stream-aggregation-config).
-
-Both input and output metric names can be modified if needed via relabeling according to [these docs](#relabeling).
-
-It is possible to leave the original metric name after the aggregation by specifying `keep_metric_names: true` option at [stream aggregation config](#stream-aggregation-config).
-The `keep_metric_names` option can be used if only a single output is set in [`outputs` list](#aggregation-outputs).
-
-## Relabeling
-
-It is possible to apply [arbitrary relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) to input and output metrics
-during stream aggregation via `input_relabel_configs` and `output_relabel_configs` options in [stream aggregation config](#stream-aggregation-config).
-
-Relabeling rules inside `input_relabel_configs` are applied to samples matching the `match` filters before optional [deduplication](#deduplication).
-Relabeling rules inside `output_relabel_configs` are applied to aggregated samples before sending them to the remote storage.
-
-For example, the following config removes the `:1m_sum_samples` suffix added [to the output metric name](#output-metric-names):
-
-```yaml
-- interval: 1m
-  outputs: [sum_samples]
-  output_relabel_configs:
-  - source_labels: [__name__]
-    target_label: __name__
-    regex: "(.+):.+"
-```
-
-Another option to remove the suffix, which is added by stream aggregation, is to add `keep_metric_names: true` to the config:
-
-```yaml
-- interval: 1m
-  outputs: [sum_samples]
-  keep_metric_names: true
-```
-
-See also [dropping unneeded labels](#dropping-unneeded-labels).
+# Configuration
 
+Stream aggregation can be configured via the following command-line flags:
 
-## Dropping unneeded labels
+- `-streamAggr.config` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/)
+  and at [vmagent](https://docs.victoriametrics.com/vmagent/).
+- `-remoteWrite.streamAggr.config` at [vmagent](https://docs.victoriametrics.com/vmagent/) only. This flag can be specified individually
+  per each `-remoteWrite.url`, so the aggregation happens independently per each remote storage destination.
+  This allows writing different aggregates to different remote storage systems.
 
-If you need dropping some labels from input samples before [input relabeling](#relabeling), [de-duplication](#deduplication)
-and [stream aggregation](#aggregation-outputs), then the following options exist:
+These flags must point to a file containing [stream aggregation config](#stream-aggregation-config).
+The file may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
 
-- To specify comma-separated list of label names to drop in `-streamAggr.dropInputLabels` command-line flag
-  or via `-remoteWrite.streamAggr.dropInputLabels` individually per each `-remoteWrite.url`.
-  For example, `-streamAggr.dropInputLabels=replica,az` instructs to drop `replica` and `az` labels from input samples
-  before applying de-duplication and stream aggregation.
+By default, the following data is written to the storage when stream aggregation is enabled:
 
-- To specify `drop_input_labels` list with the labels to drop in [stream aggregation config](#stream-aggregation-config).
-  For example, the following config drops `replica` label from input samples with the name `process_resident_memory_bytes`
-  before calculating the average over one minute:
+- the aggregated samples;
+- the raw input samples, which didn't match any `match` option in the provided [config](#stream-aggregation-config).
 
-  ```yaml
-  - match: process_resident_memory_bytes
-    interval: 1m
-    drop_input_labels: [replica]
-    outputs: [avg]
-    keep_metric_names: true
-  ```
+This behaviour can be changed via the following command-line flags:
 
-Typical use case is to drop `replica` label from samples, which are received from high availability replicas.
+- `-streamAggr.keepInput` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) 
+  and [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
+  `-remoteWrite.streamAggr.keepInput` flag can be specified individually per each `-remoteWrite.url`.
+  If one of these flags is set, then all the input samples are written to the storage alongside the aggregated samples.
+- `-streamAggr.dropInput` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) 
+  and [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
+  `-remoteWrite.streamAggr.dropInput` flag can be specified individually per each `-remoteWrite.url`.
+  If one of these flags are set, then all the input samples are dropped, while only the aggregated samples are written to the storage.
 
 ## Aggregation outputs
 
@@ -623,7 +433,7 @@ See also:
 ### histogram_bucket
 
 `histogram_bucket` returns [VictoriaMetrics histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
-  for the input [sample values](https://docs.victoriametrics.com/keyconcepts/#raw-samples) over the given `interval`.
+for the input [sample values](https://docs.victoriametrics.com/keyconcepts/#raw-samples) over the given `interval`.
 `histogram_bucket` makes sense only for aggregating [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 See how to aggregate regular histograms [here](#aggregating-histograms).
 
@@ -958,47 +768,6 @@ See also:
 - [max](#max)
 - [min](#min)
 
-
-## Aggregating by labels
-
-All the labels for the input metrics are preserved by default in the output metrics. For example,
-the input metric `foo{app="bar",instance="host1"}` results to the output metric `foo:1m_sum_samples{app="bar",instance="host1"}`
-when the following [stream aggregation config](#stream-aggregation-config) is used:
-
-```yaml
-- interval: 1m
-  outputs: [sum_samples]
-```
-
-The input labels can be removed via `without` list specified in the config. For example, the following config
-removes the `instance` label from output metrics by summing input samples across all the instances:
-
-```yaml
-- interval: 1m
-  without: [instance]
-  outputs: [sum_samples]
-```
-
-In this case the `foo{app="bar",instance="..."}` input metrics are transformed into `foo:1m_without_instance_sum_samples{app="bar"}`
-output metric according to [output metric naming](#output-metric-names).
-
-It is possible specifying the exact list of labels in the output metrics via `by` list.
-For example, the following config sums input samples by the `app` label:
-
-```yaml
-- interval: 1m
-  by: [app]
-  outputs: [sum_samples]
-```
-
-In this case the `foo{app="bar",instance="..."}` input metrics are transformed into `foo:1m_by_app_sum_samples{app="bar"}`
-output metric according to [output metric naming](#output-metric-names).
-
-The labels used in `by` and `without` lists can be modified via `input_relabel_configs` section - see [these docs](#relabeling).
-
-See also [aggregation outputs](#aggregation-outputs).
-
-
 ## Stream aggregation config
 
 Below is the format for stream aggregation config file, which may be referred via `-streamAggr.config` command-line flag at
@@ -1141,15 +910,241 @@ support the following approaches for hot reloading stream aggregation configs fr
 
 * By sending HTTP request to `/-/reload` endpoint (e.g. `http://vmagent:8429/-/reload` or `http://victoria-metrics:8428/-/reload).
 
+# Routing
+
+[Single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) supports relabeling,
+deduplication and stream aggregation for all the received data, scraped or pushed. 
+The processed data is then stored in local storage and **can't be forwarded further**.
+
+[vmagent](https://docs.victoriametrics.com/vmagent/) supports relabeling, deduplication and stream aggregation for all 
+the received data, scraped or pushed. Then, the collected data will be forwarded to specified `-remoteWrite.url` destinations.
+The data processing order is the following:
+
+1. all the received data is relabeled according to the specified [`-remoteWrite.relabelConfig`](https://docs.victoriametrics.com/vmagent/#relabeling) (if it is set)
+1. all the received data is deduplicated according to specified [`-streamAggr.dedupInterval`](https://docs.victoriametrics.com/stream-aggregation/#deduplication)
+   (if it is set to duration bigger than 0)
+1. all the received data is aggregated according to specified [`-streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration) (if it is set)
+1. the resulting data is then replicated to each `-remoteWrite.url`
+1. data sent to each `-remoteWrite.url` can be additionally relabeled according to the corresponding `-remoteWrite.urlRelabelConfig` (set individually per URL)
+1. data sent to each `-remoteWrite.url` can be additionally deduplicated according to the corresponding `-remoteWrite.streamAggr.dedupInterval` (set individually per URL)
+1. data sent to each `-remoteWrite.url` can be additionally aggregated according to the corresponding `-remoteWrite.streamAggr.config` (set individually per URL)
+   It isn't recommended using `-streamAggr.config` and `-remoteWrite.streamAggr.config` simultaneously, unless you understand the complications.
+
+Typical scenarios for data routing with `vmagent`:
+
+1. **Aggregate incoming data and replicate to N destinations**. Specify [`-streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration) command-line flag
+   to aggregate the incoming data before replicating it to all the configured `-remoteWrite.url` destinations.
+2. **Individually aggregate incoming data for each destination**. Specify [`-remoteWrite.streamAggr.config`](https://docs.victoriametrics.com/stream-aggregation/#configuration)
+   command-line flag for each `-remoteWrite.url` destination. [Relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) via `-remoteWrite.urlRelabelConfig`
+   can be used for routing only the selected metrics to each `-remoteWrite.url` destination.
+
+# Deduplication
+
+[vmagent](https://docs.victoriametrics.com/vmagent/) supports online [de-duplication](https://docs.victoriametrics.com/#deduplication) of samples
+before sending them to the configured `-remoteWrite.url`. The de-duplication can be enabled via the following options:
+
+- By specifying the desired de-duplication interval via `-streamAggr.dedupInterval` command-line flag for all received data 
+  or via `-remoteWrite.streamAggr.dedupInterval` command-line flag for the particular `-remoteWrite.url` destination.
+  For example, `./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -remoteWrite.streamAggr.dedupInterval=30s` instructs `vmagent` to leave
+  only the last sample per each seen [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) per every 30 seconds.
+  The de-deduplication is performed after applying [relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) and
+  before performing the aggregation.
+
+- By specifying `dedup_interval` option individually per each [stream aggregation config](#stream-aggregation-config) 
+  in `-remoteWrite.streamAggr.config` or `-streamAggr.config` configs.
+
+[Single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) supports two types of de-duplication:
+- After storing the duplicate samples to local storage. See [`-dedup.minScrapeInterval`](https://docs.victoriametrics.com/#deduplication) command-line option.
+- Before storing the duplicate samples to local storage. This type of de-duplication can be enabled via the following options:
+  - By specifying the desired de-duplication interval via `-streamAggr.dedupInterval` command-line flag.
+    For example, `./victoria-metrics -streamAggr.dedupInterval=30s` instructs VictoriaMetrics to leave only the last sample per each
+    seen [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) per every 30 seconds.
+    The de-duplication is performed after applying `-relabelConfig` [relabeling](https://docs.victoriametrics.com/#relabeling).
+
+  - By specifying `dedup_interval` option individually per each [stream aggregation config](#stream-aggregation-config) at `-streamAggr.config`.
+
+It is possible to drop the given labels before applying the de-duplication. See [these docs](#dropping-unneeded-labels).
+
+The online de-duplication uses the same logic as [`-dedup.minScrapeInterval` command-line flag](https://docs.victoriametrics.com/#deduplication) at VictoriaMetrics.
+
+# Relabeling
+
+It is possible to apply [arbitrary relabeling](https://docs.victoriametrics.com/vmagent/#relabeling) to input and output metrics
+during stream aggregation via `input_relabel_configs` and `output_relabel_configs` options in [stream aggregation config](#stream-aggregation-config).
+
+Relabeling rules inside `input_relabel_configs` are applied to samples matching the `match` filters before optional [deduplication](#deduplication).
+Relabeling rules inside `output_relabel_configs` are applied to aggregated samples before sending them to the remote storage.
+
+For example, the following config removes the `:1m_sum_samples` suffix added [to the output metric name](#output-metric-names):
+
+```yaml
+- interval: 1m
+  outputs: [sum_samples]
+  output_relabel_configs:
+  - source_labels: [__name__]
+    target_label: __name__
+    regex: "(.+):.+"
+```
+
+Another option to remove the suffix, which is added by stream aggregation, is to add `keep_metric_names: true` to the config:
+
+```yaml
+- interval: 1m
+  outputs: [sum_samples]
+  keep_metric_names: true
+```
+
+See also [dropping unneeded labels](#dropping-unneeded-labels).
+
+# Advanced usage
+
+## Ignoring old samples
+
+By default, all the input samples are taken into account during stream aggregation. If samples with old timestamps 
+outside the current [aggregation interval](#stream-aggregation-config) must be ignored, then the following options can be used:
+
+- To pass `-streamAggr.ignoreOldSamples` command-line flag to [single-node VictoriaMetrics](https://docs.victoriametrics.com/)
+  or to [vmagent](https://docs.victoriametrics.com/vmagent/). At [vmagent](https://docs.victoriametrics.com/vmagent/)
+  `-remoteWrite.streamAggr.ignoreOldSamples` flag can be specified individually per each `-remoteWrite.url`.
+  This enables ignoring old samples for all the [aggregation configs](#stream-aggregation-config).
+
+- To set `ignore_old_samples: true` option at the particular [aggregation config](#stream-aggregation-config).
+  This enables ignoring old samples for that particular aggregation config.
+
+## Ignore aggregation intervals on start
+
+Streaming aggregation results may be incorrect for some time after the restart of [vmagent](https://docs.victoriametrics.com/vmagent/)
+or [single-node VictoriaMetrics](https://docs.victoriametrics.com/) until all the buffered [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+are sent from remote sources to the `vmagent` or single-node VictoriaMetrics via [supported data ingestion protocols](https://docs.victoriametrics.com/vmagent/#how-to-push-data-to-vmagent).
+In this case it may be a good idea to drop the aggregated data during the first `N` [aggregation intervals](#stream-aggregation-config)
+just after the restart of `vmagent` or single-node VictoriaMetrics. This can be done via the following options:
+
+- The `-streamAggr.ignoreFirstIntervals=N` command-line flag at `vmagent` and single-node VictoriaMetrics. This flag instructs skipping the first `N`
+  [aggregation intervals](#stream-aggregation-config) just after the restart across all the [configured stream aggregation configs](#configuration).
+
+  The `-remoteWrite.streamAggr.ignoreFirstIntervals` command-line flag can be specified individually per each `-remoteWrite.url` at [vmagent](https://docs.victoriametrics.com/vmagent/).
+
+- The `ignore_first_intervals: N` option at the particular [aggregation config](#stream-aggregation-config).
+
+See also:
 
-## Troubleshooting
+- [Flush time alignment](#flush-time-alignment)
+- [Ignoring old samples](#ignoring-old-samples)
+
+## Flush time alignment
+
+By default, the time for aggregated data flush is aligned by the `interval` option specified in [aggregate config](#stream-aggregation-config).
+
+For example:
+
+- if `interval: 1m` is set, then the aggregated data is flushed to the storage at the end of every minute
+- if `interval: 1h` is set, then the aggregated data is flushed to the storage at the end of every hour
+
+If you do not need such an alignment, then set `no_align_flush_to_interval: true` option in the [aggregate config](#stream-aggregation-config).
+In this case aggregated data flushes will be aligned to the `vmagent` start time or to [config reload](#configuration-update) time.
+
+The aggregated data on the first and the last interval is dropped during `vmagent` start, restart or [config reload](#configuration-update),
+since the first and the last aggregation intervals are incomplete, so they usually contain incomplete confusing data.
+If you need preserving the aggregated data on these intervals, then set `flush_on_shutdown: true` option in the [aggregate config](#stream-aggregation-config).
+
+See also:
+
+- [Ignore aggregation intervals on start](#ignore-aggregation-intervals-on-start)
+- [Ignoring old samples](#ignoring-old-samples)
+
+## Output metric names
+
+Output metric names for stream aggregation are constructed according to the following pattern:
+
+```text
+<metric_name>:<interval>[_by_<by_labels>][_without_<without_labels>]_<output>
+```
+
+- `<metric_name>` is the original metric name.
+- `<interval>` is the interval specified in the [stream aggregation config](#stream-aggregation-config).
+- `<by_labels>` is `_`-delimited sorted list of `by` labels specified in the [stream aggregation config](#stream-aggregation-config).
+  If the `by` list is missing in the config, then the `_by_<by_labels>` part isn't included in the output metric name.
+- `<without_labels>` is an optional `_`-delimited sorted list of `without` labels specified in the [stream aggregation config](#stream-aggregation-config).
+  If the `without` list is missing in the config, then the `_without_<without_labels>` part isn't included in the output metric name.
+- `<output>` is the aggregate used for constructing the output metric. The aggregate name is taken from the `outputs` list
+  at the corresponding [stream aggregation config](#stream-aggregation-config).
+
+Both input and output metric names can be modified if needed via relabeling according to [these docs](#relabeling).
+
+It is possible to leave the original metric name after the aggregation by specifying `keep_metric_names: true` option at [stream aggregation config](#stream-aggregation-config).
+The `keep_metric_names` option can be used if only a single output is set in [`outputs` list](#aggregation-outputs).
+
+## Aggregating by labels
+
+All the labels for the input metrics are preserved by default in the output metrics. For example,
+the input metric `foo{app="bar",instance="host1"}` results to the output metric `foo:1m_sum_samples{app="bar",instance="host1"}`
+when the following [stream aggregation config](#stream-aggregation-config) is used:
+
+```yaml
+- interval: 1m
+  outputs: [sum_samples]
+```
+
+The input labels can be removed via `without` list specified in the config. For example, the following config
+removes the `instance` label from output metrics by summing input samples across all the instances:
+
+```yaml
+- interval: 1m
+  without: [instance]
+  outputs: [sum_samples]
+```
+
+In this case the `foo{app="bar",instance="..."}` input metrics are transformed into `foo:1m_without_instance_sum_samples{app="bar"}`
+output metric according to [output metric naming](#output-metric-names).
+
+It is possible specifying the exact list of labels in the output metrics via `by` list.
+For example, the following config sums input samples by the `app` label:
+
+```yaml
+- interval: 1m
+  by: [app]
+  outputs: [sum_samples]
+```
+
+In this case the `foo{app="bar",instance="..."}` input metrics are transformed into `foo:1m_by_app_sum_samples{app="bar"}`
+output metric according to [output metric naming](#output-metric-names).
+
+The labels used in `by` and `without` lists can be modified via `input_relabel_configs` section - see [these docs](#relabeling).
+
+See also [aggregation outputs](#aggregation-outputs).
+
+## Dropping unneeded labels
+
+If you need dropping some labels from input samples before [input relabeling](#relabeling), [de-duplication](#deduplication)
+and [stream aggregation](#aggregation-outputs), then the following options exist:
+
+- To specify comma-separated list of label names to drop in `-streamAggr.dropInputLabels` command-line flag
+  or via `-remoteWrite.streamAggr.dropInputLabels` individually per each `-remoteWrite.url`.
+  For example, `-streamAggr.dropInputLabels=replica,az` instructs to drop `replica` and `az` labels from input samples
+  before applying de-duplication and stream aggregation.
+
+- To specify `drop_input_labels` list with the labels to drop in [stream aggregation config](#stream-aggregation-config).
+  For example, the following config drops `replica` label from input samples with the name `process_resident_memory_bytes`
+  before calculating the average over one minute:
+
+  ```yaml
+  - match: process_resident_memory_bytes
+    interval: 1m
+    drop_input_labels: [replica]
+    outputs: [avg]
+    keep_metric_names: true
+  ```
+
+Typical use case is to drop `replica` label from samples, which are received from high availability replicas.
+
+# Troubleshooting
 
 - [Unexpected spikes for `total` or `increase` outputs](#staleness).
 - [Lower than expected values for `total_prometheus` and `increase_prometheus` outputs](#staleness).
 - [High memory usage and CPU usage](#high-resource-usage).
 - [Unexpected results in vmagent cluster mode](#cluster-mode).
 
-### Staleness
+## Staleness
 
 The following outputs track the last seen per-series values in order to properly calculate output values:
 
@@ -1177,7 +1172,7 @@ These issues can be fixed in the following ways:
 - By specifying the `staleness_interval` option at [stream aggregation config](#stream-aggregation-config), so it covers the expected
   delays in data ingestion pipelines. By default, the `staleness_interval` equals to `2 x interval`.
 
-### High resource usage
+## High resource usage
 
 The following solutions can help reducing memory usage and CPU usage during streaming aggregation:
 
@@ -1187,7 +1182,7 @@ The following solutions can help reducing memory usage and CPU usage during stre
 - To generate lower number of output time series by using less specific [`by` list](#aggregating-by-labels) or more specific [`without` list](#aggregating-by-labels).
 - To drop unneeded long labels in input samples via [input_relabel_configs](#relabeling).
 
-### Cluster mode
+## Cluster mode
 
 If you use [vmagent in cluster mode](https://docs.victoriametrics.com/vmagent/#scraping-big-number-of-targets) for streaming aggregation
 then be careful when using [`by` or `without` options](#aggregating-by-labels) or when modifying sample labels
@@ -1195,10 +1190,75 @@ via [relabeling](#relabeling), since incorrect usage may result in duplicates an
 
 For example, if more than one `vmagent` instance calculates [increase](#increase) for `http_requests_total` metric
 with `by: [path]` option, then all the `vmagent` instances will aggregate samples to the same set of time series with different `path` labels.
-The proper fix would be [adding an unique label](https://docs.victoriametrics.com/vmagent/#adding-labels-to-metrics) for all the output samples
+The proper fix would be [adding a unique label](https://docs.victoriametrics.com/vmagent/#adding-labels-to-metrics) for all the output samples
 produced by each `vmagent`, so they are aggregated into distinct sets of [time series](https://docs.victoriametrics.com/keyconcepts/#time-series).
 These time series then can be aggregated later as needed during querying.
 
 If `vmagent` instances run in Docker or Kubernetes, then you can refer `POD_NAME` or `HOSTNAME` environment variables
-as an unique label value per each `vmagent` via `-remoteWrite.label=vmagent=%{HOSTNAME}` command-line flag.
+as a unique label value per each `vmagent` via `-remoteWrite.label=vmagent=%{HOSTNAME}` command-line flag.
 See [these docs](https://docs.victoriametrics.com/#environment-variables) on how to refer environment variables in VictoriaMetrics components.
+
+## Common mistakes
+
+### Put aggregator behind load balancer
+
+When configuring the aggregation rule, make sure that `vmagent` receives all the required data to satisfy the `match` rule.
+If traffic to the vmagent goes through the load balancer, it could happen that vmagent will be receiving only fraction of the data
+and produce incomplete aggregations.
+
+To keep aggregation results consistent, make sure that vmagent receives all the required data for aggregation. In case if you need to 
+split the load across multiple vmagents, try sharding the traffic among them via metric names or labels.
+For example, see how vmagent could consistently [shard data across remote write destinations](https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages)
+via `-remoteWrite.shardByURL.labels` or `-remoteWrite.shardByURL.ignoreLabels` cmd-line flags.
+
+### Create aggregator per each recording rule
+
+Stream aggregation can be used as alternative for [recording rules](#recording-rules-alternative).
+But creating an aggregation rule per each recording rule can lead to elevated resource usage on the vmagent,
+because the ingestion stream should be matched against every configured aggregation rule.
+
+To optimize this, we recommend merging together aggregations which only differ in match expressions. 
+For example, let's see the following list of recording rules:
+
+```yaml
+- expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)
+  record: instance:node_cpu:rate:sum
+- expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
+  record: instance:node_network_receive_bytes:rate:sum
+- expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
+  record: instance:node_network_transmit_bytes:rate:sum
+```
+
+These rules can be effectively converted into a single aggregation rule:
+
+```yaml
+- match:
+  - node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}
+  - node_network_receive_bytes_total
+  - node_network_transmit_bytes_total
+  interval: 3m
+  outputs: [rate_sum]  
+  by:
+  - instance
+  output_relabel_configs:
+    - source_labels: [__name__]
+      target_label: __name__
+      regex: "(.+):.+"
+      replacement: "instance:$1:rate:sum"
+```
+
+**Note**: having separate aggregator for a certain `match` expression can only be justified when aggregator cannot keep up with all
+the data pushed to an aggregator within an aggregation interval.
+
+### Use identical --remoteWrite.streamAggr.config for all remote writes
+
+Each specified `-remoteWrite.streamAggr.config` aggregation config is processed independently on the copy of the data stream.
+So if you want to aggregate incoming data and replicate it across multiple destinations, it would be more efficient
+to use a global `-streamAggr.config` instead. In this way, vmagent will perform aggregation only once and then will replicate it
+across multiple `-remoteWrite.url`.
+
+### Use aggregated metrics like original ones
+
+Stream aggregation allows keeping original metric names after aggregation by using `keep_metric_names` setting.
+But the "meaning" of aggregated metrics is usually different to original ones after the aggregation.
+Make sure that you updated queries in your alerting rules and dashboards accordingly if you used `keep_metric_names` setting.
\ No newline at end of file
diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go
index 45918d9b8..05e183c9e 100644
--- a/lib/logstorage/parser.go
+++ b/lib/logstorage/parser.go
@@ -714,13 +714,6 @@ func ParseQuery(s string) (*Query, error) {
 func ParseQueryAtTimestamp(s string, timestamp int64) (*Query, error) {
 	lex := newLexerAtTimestamp(s, timestamp)
 
-	// Verify the first token doesn't match pipe names.
-	firstToken := strings.ToLower(lex.rawToken)
-	if _, ok := pipeNames[firstToken]; ok {
-		return nil, fmt.Errorf("the query [%s] cannot start with pipe - it must start with madatory filter; see https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax; "+
-			"if the filter isn't missing, then please put the first word of the filter into quotes: %q", s, firstToken)
-	}
-
 	q, err := parseQuery(lex)
 	if err != nil {
 		return nil, err
@@ -759,9 +752,17 @@ func parseQuery(lex *lexer) (*Query, error) {
 }
 
 func parseFilter(lex *lexer) (filter, error) {
-	if lex.isKeyword("|", "") {
+	if lex.isKeyword("|", ")", "") {
 		return nil, fmt.Errorf("missing query")
 	}
+
+	// Verify the first token in the filter doesn't match pipe names.
+	firstToken := strings.ToLower(lex.rawToken)
+	if _, ok := pipeNames[firstToken]; ok {
+		return nil, fmt.Errorf("query filter cannot start with pipe keyword %q; see https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax; "+
+			"please put the first word of the filter into quotes", firstToken)
+	}
+
 	fo, err := parseFilterOr(lex, "")
 	if err != nil {
 		return nil, err
diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go
index a8dcff2a6..d2af609c1 100644
--- a/lib/logstorage/pipe.go
+++ b/lib/logstorage/pipe.go
@@ -317,7 +317,7 @@ var pipeNames = func() map[string]struct{} {
 		"replace",
 		"replace_regexp",
 		"sort", "order",
-		"stats",
+		"stats", "by",
 		"stream_context",
 		"top",
 		"uniq",


Test patch:
diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go
index 77e46ca46..7b22fa44b 100644
--- a/lib/logstorage/parser_test.go
+++ b/lib/logstorage/parser_test.go
@@ -1196,6 +1196,11 @@ func TestParseQuerySuccess(t *testing.T) {
 	// filter pipe
 	f(`* | filter error ip:12.3.4.5 or warn`, `* | filter error ip:12.3.4.5 or warn`)
 	f(`foo | stats by (host) count() logs | filter logs:>50 | sort by (logs desc) | limit 10`, `foo | stats by (host) count(*) as logs | filter logs:>50 | sort by (logs desc) | limit 10`)
+	f(`* | error`, `* | filter error`)
+	f(`* | "by"`, `* | filter "by"`)
+	f(`* | "stats"`, `* | filter "stats"`)
+	f(`* | "count"`, `* | filter "count"`)
+	f(`* | foo:bar AND baz:<10`, `* | filter foo:bar baz:<10`)
 
 	// extract pipe
 	f(`* | extract "foo<bar>baz"`, `* | extract "foo<bar>baz"`)
@@ -1237,7 +1242,7 @@ func TestParseQueryFailure(t *testing.T) {
 		t.Helper()
 		q, err := ParseQuery(s)
 		if q != nil {
-			t.Fatalf("expecting nil result; got %s", q)
+			t.Fatalf("expecting nil result; got [%s]", q)
 		}
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
@@ -1635,6 +1640,9 @@ func TestParseQueryFailure(t *testing.T) {
 	// stats result names identical to by fields
 	f(`foo | stats by (x) count() x`)
 
+	// missing stats function
+	f(`foo | by (bar)`)
+
 	// invalid sort pipe
 	f(`foo | sort bar`)
 	f(`foo | sort by`)
@@ -1669,6 +1677,14 @@ func TestParseQueryFailure(t *testing.T) {
 	f(`foo | filter (`)
 	f(`foo | filter )`)
 
+	f(`foo | filter stats`)
+	f(`foo | filter fields`)
+	f(`foo | filter by`)
+	f(`foo | count`)
+	f(`foo | filter count`)
+	f(`foo | (`)
+	f(`foo | )`)
+
 	// invalid extract pipe
 	f(`foo | extract`)
 	f(`foo | extract bar`)