Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1622 (2021-09-16 06:44:47)

vmalert: add new metric `vmalert_remotewrite_flush_duration_seconds`

codecov[bot]: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
> Merging [#1622](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) (ca121e6) into [master](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/26cd0d36b4764d730e87fb5c48667ee2b4aa02fe?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) (26cd0d3) will **decrease** coverage by `0.00%`.
> The diff coverage is `57.14%`.

[![Impacted file tree graph](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622/graphs/tree.svg?width=650&height=150&src=pr&token=YND8dYVcXn&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)

```diff
@@            Coverage Diff             @@
##           master    #1622      +/-   ##
==========================================
- Coverage   60.59%   60.59%   -0.01%     
==========================================
  Files         234      234              
  Lines       38686    38688       +2     
==========================================
  Hits        23443    23443              
- Misses      13688    13694       +6     
+ Partials     1555     1551       -4     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Coverage Δ | |
|---|---|---|
| [app/vmalert/remotewrite/remotewrite.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvcmVtb3Rld3JpdGUvcmVtb3Rld3JpdGUuZ28=) | `64.82% <57.14%> (+1.89%)` | :arrow_up: |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `65.56% <0.00%> (-0.97%)` | :arrow_down: |
| [lib/storage/partition.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-bGliL3N0b3JhZ2UvcGFydGl0aW9uLmdv) | `65.72% <0.00%> (-0.17%)` | :arrow_down: |
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `86.44% <0.00%> (+2.40%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics). Last update [26cd0d3...ca121e6](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/1622?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go
index eaac3a4bb..30d948224 100644
--- a/app/vmalert/remotewrite/remotewrite.go
+++ b/app/vmalert/remotewrite/remotewrite.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strings"
 	"sync"
 	"time"
@@ -92,6 +93,10 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Transport == nil {
 		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
 	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.WriteTimeout,
@@ -106,10 +111,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 		disablePathAppend: cfg.DisablePathAppend,
 	}
-	cc := defaultConcurrency
-	if cfg.Concurrency > 0 {
-		cc = cfg.Concurrency
-	}
+
 	for i := 0; i < cc; i++ {
 		c.run(ctx)
 	}
@@ -182,10 +184,11 @@ func (c *Client) run(ctx context.Context) {
 }
 
 var (
-	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -196,6 +199,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		return
 	}
 	defer prompbmarshal.ResetWriteRequest(wr)
+	defer bufferFlushDuration.UpdateDuration(time.Now())
 
 	data, err := wr.Marshal()
 	if err != nil {
@@ -237,7 +241,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		}
 	}
 	if !c.disablePathAppend {
-		req.URL.Path += writePath
+		req.URL.Path = path.Join(req.URL.Path, writePath)
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
diff --git a/app/vmselect/main.go b/app/vmselect/main.go
index 00654edaa..a45342047 100644
--- a/app/vmselect/main.go
+++ b/app/vmselect/main.go
@@ -198,7 +198,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	}
-
+	if strings.HasPrefix(path, "/functions") {
+		graphiteFunctionsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "%s", `{}`)
+		return true
+	}
 	switch path {
 	case "/api/v1/query":
 		queryRequests.Inc()
@@ -544,6 +549,8 @@ var (
 	graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`)
 	graphiteTagsDelSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`)
 
+	graphiteFunctionsRequests = metrics.NewCounter(`vm_http_request_total{path="/functions"}`)
+
 	rulesRequests          = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
 	alertsRequests         = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index ac25cc1cb..93323658d 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -98,7 +98,7 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		return err
 	})
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("error during sending data to remote client: %w", err)
 	}
 	if err := bw.Flush(); err != nil {
 		return err
@@ -175,7 +175,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during exporting data to csv: %w", err)
+		return fmt.Errorf("error during sending the exported csv data to remote client: %w", err)
 	}
 	return nil
 }
@@ -244,10 +244,10 @@ func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 		return err
 	})
 	if err != nil {
-		return err
+		return fmt.Errorf("error during sending native data to remote client: %w", err)
 	}
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("error during flushing native data to remote client: %w", err)
 	}
 	return nil
 }
@@ -414,7 +414,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etf []storage.TagFil
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("error during sending the data to remote client: %w", err)
 	}
 	return nil
 }
@@ -538,7 +538,7 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 	defer bufferedwriter.Put(bw)
 	WriteLabelValuesResponse(bw, labelValues)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("canot flush label values to remote client: %w", err)
 	}
 	return nil
 }
@@ -600,7 +600,7 @@ func labelValuesWithMatches(labelName string, matches []string, etf []storage.Ta
 			return nil
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error when data fetching: %w", err)
+			return nil, fmt.Errorf("cannot fetch label values from storage: %w", err)
 		}
 	}
 	labelValues := make([]string, 0, len(m))
@@ -627,7 +627,7 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	defer bufferedwriter.Put(bw)
 	WriteLabelsCountResponse(bw, labelEntries)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send labels count response to remote client: %w", err)
 	}
 	return nil
 }
@@ -695,7 +695,7 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	defer bufferedwriter.Put(bw)
 	WriteTSDBStatusResponse(bw, status)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send tsdb status response to remote client: %w", err)
 	}
 	return nil
 }
@@ -789,7 +789,7 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	defer bufferedwriter.Put(bw)
 	WriteLabelsResponse(bw, labels)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send labels response to remote client: %w", err)
 	}
 	return nil
 }
@@ -838,7 +838,7 @@ func labelsWithMatches(matches []string, etf []storage.TagFilter, start, end int
 			return nil
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error when data fetching: %w", err)
+			return nil, fmt.Errorf("cannot fetch labels from storage: %w", err)
 		}
 	}
 	labels := make([]string, 0, len(m))
@@ -865,7 +865,7 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	defer bufferedwriter.Put(bw)
 	WriteSeriesCountResponse(bw, n)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send series count response to remote client: %w", err)
 	}
 	return nil
 }
@@ -957,11 +957,11 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	// WriteSeriesResponse must consume all the data from resultsCh.
 	WriteSeriesResponse(bw, resultsCh)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot flush series response to remote client: %w", err)
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("cannot send series response to remote client: %w", err)
 	}
 	return nil
 }
@@ -1075,7 +1075,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	defer bufferedwriter.Put(bw)
 	WriteQueryResponse(bw, result)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
 	return nil
 }
@@ -1168,7 +1168,7 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	defer bufferedwriter.Put(bw)
 	WriteQueryRangeResponse(bw, result)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}
 	return nil
 }
@@ -1348,7 +1348,7 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	defer bufferedwriter.Put(bw)
 	querystats.WriteJSONQueryStats(bw, topN, maxLifetime)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send query stats response to client: %w", err)
 	}
 	return nil
 }
diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go
index 78f549fb7..744ffbf38 100644
--- a/app/vmselect/promql/aggr.go
+++ b/app/vmselect/promql/aggr.go
@@ -45,6 +45,8 @@ var aggrFuncs = map[string]aggrFunc{
 	"bottomk_avg":    newAggrFuncRangeTopK(avgValue, true),
 	"bottomk_median": newAggrFuncRangeTopK(medianValue, true),
 	"any":            aggrFuncAny,
+	"mad":            newAggrFunc(aggrFuncMAD),
+	"outliers_mad":   aggrFuncOutliersMAD,
 	"outliersk":      aggrFuncOutliersK,
 	"mode":           newAggrFunc(aggrFuncMode),
 	"zscore":         aggrFuncZScore,
@@ -485,7 +487,6 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 				ts.Values[i] = (v - avg) / stddev
 			}
 		}
-
 		// Remove MetricGroup from all the tss.
 		for _, ts := range tss {
 			ts.MetricName.ResetMetricGroup()
@@ -811,31 +812,64 @@ func medianValue(values []float64) float64 {
 	return value
 }
 
-func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
+func aggrFuncMAD(tss []*timeseries) []*timeseries {
+	// Calculate medians for each point across tss.
+	medians := getPerPointMedians(tss)
+	// Calculate MAD values multipled by tolerance for each point across tss.
+	// See https://en.wikipedia.org/wiki/Median_absolute_deviation
+	mads := getPerPointMADs(tss, medians)
+	tss[0].Values = append(tss[0].Values[:0], mads...)
+	return tss[:1]
+}
+
+func aggrFuncOutliersMAD(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
 		return nil, err
 	}
-	ks, err := getScalar(args[0], 0)
+	tolerances, err := getScalar(args[0], 0)
 	if err != nil {
 		return nil, err
 	}
 	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		// Calculate medians for each point across tss.
-		medians := make([]float64, len(ks))
-		h := histogram.GetFast()
-		for n := range ks {
-			h.Reset()
-			for j := range tss {
-				v := tss[j].Values[n]
-				if !math.IsNaN(v) {
-					h.Update(v)
+		medians := getPerPointMedians(tss)
+		// Calculate MAD values multipled by tolerance for each point across tss.
+		// See https://en.wikipedia.org/wiki/Median_absolute_deviation
+		mads := getPerPointMADs(tss, medians)
+		for n := range mads {
+			mads[n] *= tolerances[n]
+		}
+		// Leave only time series with at least a single peak above the MAD multiplied by tolerance.
+		tssDst := tss[:0]
+		for _, ts := range tss {
+			values := ts.Values
+			for n, v := range values {
+				ad := math.Abs(v - medians[n])
+				mad := mads[n]
+				if ad > mad {
+					tssDst = append(tssDst, ts)
+					break
 				}
 			}
-			medians[n] = h.Quantile(0.5)
 		}
-		histogram.PutFast(h)
+		return tssDst
+	}
+	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
+}
 
+func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
+	args := afa.args
+	if err := expectTransformArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	ks, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, err
+	}
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+		// Calculate medians for each point across tss.
+		medians := getPerPointMedians(tss)
 		// Return topK time series with the highest variance from median.
 		f := func(values []float64) float64 {
 			sum2 := float64(0)
@@ -850,6 +884,44 @@ func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
 	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 }
 
+func getPerPointMedians(tss []*timeseries) []float64 {
+	if len(tss) == 0 {
+		logger.Panicf("BUG: expecting non-empty tss")
+	}
+	medians := make([]float64, len(tss[0].Values))
+	h := histogram.GetFast()
+	for n := range medians {
+		h.Reset()
+		for j := range tss {
+			v := tss[j].Values[n]
+			if !math.IsNaN(v) {
+				h.Update(v)
+			}
+		}
+		medians[n] = h.Quantile(0.5)
+	}
+	histogram.PutFast(h)
+	return medians
+}
+
+func getPerPointMADs(tss []*timeseries, medians []float64) []float64 {
+	mads := make([]float64, len(medians))
+	h := histogram.GetFast()
+	for n, median := range medians {
+		h.Reset()
+		for j := range tss {
+			v := tss[j].Values[n]
+			if !math.IsNaN(v) {
+				ad := math.Abs(v - median)
+				h.Update(ad)
+			}
+		}
+		mads[n] = h.Quantile(0.5)
+	}
+	histogram.PutFast(h)
+	return mads
+}
+
 func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1a9c10d56..fcfd9b298 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -24,6 +24,8 @@ sort: 15
 * FEATURE: vmui: use Prometheus-compatible query args, so `vmui` could be accessed from graph editor in Grafana. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1619). Thanks to @Loori-R.
 * FEATURE: vmselect: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8401,vmstorage2:8401`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup).
 * FEATURE: vminsert: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8400,vmstorage2:8400`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup).
+* FEATURE: add [mad(q)](https://docs.victoriametrics.com/MetricsQL.html#mad) function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). It calculates [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) for groups of points with identical timestamps across multiple time series.
+* FEATURE: add [outliers_mad(tolerance, q)](https://docs.victoriametrics.com/MetricsQL.html#outliers_mad) function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). It returns time series with peaks outside the [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) multiplied by `tolerance`.
 
 * BUGFIX: properly handle queries with multiple filters matching empty labels such as `metric{label1=~"foo|",label2="bar|"}`. This filter must match the following series: `metric`, `metric{label1="foo"}`, `metric{label2="bar"}` and `metric{label1="foo",label2="bar"}`. Previously it was matching only `metric{label1="foo",label2="bar"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1601).
 * BUGFIX: vmselect: reset connection timeouts after each request to `vmstorage`. This should prevent from `cannot read data in 0.000 seconds: unexpected EOF` warning in logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1562). Thanks to @mxlxm .
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 15aadd173..cae0a05d0 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -4,37 +4,37 @@ sort: 14
 
 # FAQ
 
-### What is the main purpose of VictoriaMetrics?
+## What is the main purpose of VictoriaMetrics?
 
 To provide the best monitoring solution.
 
 
-### Who uses VictoriaMetrics?
+## Who uses VictoriaMetrics?
 
 See [case studies](https://docs.victoriametrics.com/CaseStudies.html).
 
 
-### Which features does VictoriaMetrics have?
+## Which features does VictoriaMetrics have?
 
 See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prominent-features).
 
 
-### Are there performance comparisons with other solutions?
+## Are there performance comparisons with other solutions?
 
 Yes. See [these benchmarks](https://docs.victoriametrics.com/Articles.html#benchmarks).
 
 
-### How to start using VictoriaMetrics?
+## How to start using VictoriaMetrics?
 
 See [these docs](https://docs.victoriametrics.com/Quick-Start.html).
 
 
-### Does VictoriaMetrics support replication?
+## Does VictoriaMetrics support replication?
 
 Yes. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#replication-and-data-safety) for details.
 
 
-### Can I use VictoriaMetrics instead of Prometheus?
+## Can I use VictoriaMetrics instead of Prometheus?
 
 Yes in most cases. VictoriaMetrics can substitute Prometheus in the following aspects:
 
@@ -43,7 +43,7 @@ Yes in most cases. VictoriaMetrics can substitute Prometheus in the following as
 * Prometheus-compatible querying in Grafana is supported by VictoriaMetrics. See [these docs](https://docs.victoriametrics.com/#grafana-setup).
 
 
-### What is the difference between vmagent and Prometheus?
+## What is the difference between vmagent and Prometheus?
 
 While both [vmagent](https://docs.victoriametrics.com/vmagent.html) and Prometheus may scrape Prometheus targets (aka `/metrics` pages)
 according to the provided Prometheus-compatible [scrape configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)
@@ -66,7 +66,7 @@ and send data to multiple remote storage systems, vmagent has the following addi
   - [Prometheus remote_write proxy](https://docs.victoriametrics.com/vmagent.html#prometheus-remote_write-proxy)
 
 
-### Is it safe to enable [remote write](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) in Prometheus?
+## Is it safe to enable [remote write](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) in Prometheus?
 
 Yes. Prometheus continues writing data to local storage after enabling remote write, so all the existing local storage data
 and new data is available for querying via Prometheus as usual.
@@ -75,7 +75,7 @@ It is recommended using [vmagent](https://docs.victoriametrics.com/vmagent.html)
 and writing data to VictoriaMetrics.
 
 
-### How does VictoriaMetrics compare to other remote storage solutions for Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), etc.?
+## How does VictoriaMetrics compare to other remote storage solutions for Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), etc.?
 
 VictoriaMetrics is simpler, faster, more cost-effective and it provides [MetricsQL query language](MetricsQL) based on PromQL. The simplicity is twofold:
 - It is simpler to configure and operate. There is no need in configuring [sidecars](https://github.com/thanos-io/thanos/blob/master/docs/components/sidecar.md),
@@ -90,7 +90,7 @@ and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-writ
 VictoriaMetrics also [uses less RAM than Thanos components](https://github.com/thanos-io/thanos/issues/448).
 
 
-### What is the difference between VictoriaMetrics and [QuestDB](https://questdb.io/)?
+## What is the difference between VictoriaMetrics and [QuestDB](https://questdb.io/)?
 
 - QuestDB needs more than 20x storage space than VictoriaMetrics. This translates to higher storage costs and slower queries over historical data, which must be read from the disk.
 - QuestDB is much harder to setup and operate than VictoriaMetrics. Compare [setup instructions for QuestDB](https://questdb.io/docs/get-started/binaries) to [setup instructions for VictoriaMetrics](https://docs.victoriametrics.com/#how-to-start-victoriametrics).
@@ -101,7 +101,7 @@ VictoriaMetrics also [uses less RAM than Thanos components](https://github.com/t
 - [VictoriaMetrics supports backfilling (e.g. storing historical data) out of the box](https://docs.victoriametrics.com/#backfilling), while QuestDB provides [very limited support for backfilling](https://questdb.io/blog/2021/05/10/questdb-release-6-0-tsbs-benchmark#the-problem-with-out-of-order-data).
 
 
-### What is the difference between VictoriaMetrics and [Cortex](https://github.com/cortexproject/cortex)?
+## What is the difference between VictoriaMetrics and [Cortex](https://github.com/cortexproject/cortex)?
 
 VictoriaMetrics is similar to Cortex in the following aspects:
 - Both systems accept data from [vmagent](https://docs.victoriametrics.com/vmagent.html) or Prometheus
@@ -129,7 +129,7 @@ The main differences between Cortex and VictoriaMetrics:
   See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-time-series-data) for details.
 
 
-### What is the difference between VictoriaMetrics and [Thanos](https://github.com/thanos-io/thanos)?
+## What is the difference between VictoriaMetrics and [Thanos](https://github.com/thanos-io/thanos)?
 
 - Thanos re-uses Prometheus source code, while VictoriaMetrics is written from scratch.
 - VictoriaMetrics accepts data via [standard remote_write API for Prometheus](https://prometheus.io/docs/practices/remote_write/),
@@ -150,7 +150,7 @@ The main differences between Cortex and VictoriaMetrics:
   See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-time-series-data) for details.
 
 
-### How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)?
+## How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)?
 
 - VictoriaMetrics requires [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and it [works faster](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 - VictoriaMetrics needs lower amounts of storage space than InfluxDB on production data.
@@ -159,7 +159,7 @@ The main differences between Cortex and VictoriaMetrics:
   See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-time-series-data) for details.
 
 
-### How does VictoriaMetrics compare to [TimescaleDB](https://www.timescale.com/)?
+## How does VictoriaMetrics compare to [TimescaleDB](https://www.timescale.com/)?
 
 - TimescaleDB insists on using SQL as a query language. While SQL is more powerful than PromQL, this power is rarely required during typical TSDB usage. Real-world queries usually [look clearer and simpler when written in PromQL than in SQL](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085).
 - VictoriaMetrics requires [up to 70x less storage space comparing to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data. The gap in storage space usage can be lowered from 70x to 3x if [compression in TimescaleDB is properly configured](https://docs.timescale.com/latest/using-timescaledb/compression) (it isn't an easy task in general case :)).
@@ -167,13 +167,13 @@ The main differences between Cortex and VictoriaMetrics:
 - VictoriaMetrics accepts data in multiple popular data ingestion protocols - InfluxDB, OpenTSDB, Graphite, CSV, while TimescaleDB supports only SQL inserts.
 
 
-### Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [Thanos](https://github.com/thanos-io/thanos) or [Cortex](https://github.com/cortexproject/cortex)?
+## Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [Thanos](https://github.com/thanos-io/thanos) or [Cortex](https://github.com/cortexproject/cortex)?
 
 No. VictoriaMetrics core is written in Go from scratch by [fasthttp](https://github.com/valyala/fasthttp) [author](https://github.com/valyala).
 The architecture is [optimized for storing and querying large amounts of time series data with high cardinality](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac). VictoriaMetrics storage uses [certain ideas from ClickHouse](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282). Special thanks to [Alexey Milovidov](https://github.com/alexey-milovidov).
 
 
-### What is the pricing for VictoriaMetrics?
+## What is the pricing for VictoriaMetrics?
 
 The following versions are open source and free:
 * [Single-node version](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html).
@@ -188,7 +188,7 @@ The following commercial versions of VictoriaMetrics are planned:
 [Contact us](mailto:info@victoriametrics.com) for more information on our plans.
 
 
-### Why VictoriaMetrics doesn't support [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)?
+## Why VictoriaMetrics doesn't support [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)?
 
 Remote read API requires transferring all the raw data for all the requested metrics over the given time range. For instance,
 if a query covers 1000 metrics with 10K values each, then the remote read API had to return `1000*10K`=10M metric values to Prometheus.
@@ -199,19 +199,19 @@ So just query VictoriaMetrics directly via [Prometheus Querying API](https://doc
 or via [Prometheus datasource in Grafana](https://docs.victoriametrics.com/#grafana-setup).
 
 
-### Does VictoriaMetrics deduplicate data from Prometheus instances scraping the same targets (aka `HA pairs`)?
+## Does VictoriaMetrics deduplicate data from Prometheus instances scraping the same targets (aka `HA pairs`)?
 
 Yes. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication) for details.
 
 
-### Where is the source code of VictoriaMetrics?
+## Where is the source code of VictoriaMetrics?
 
 Source code for the following versions is available in the following places:
 * [Single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics)
 * [Cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster)
 
 
-### Does VictoriaMetrics fit for data from IoT sensors and industrial sensors?
+## Does VictoriaMetrics fit for data from IoT sensors and industrial sensors?
 
 VictoriaMetrics is able to handle data from hundreds of millions of IoT sensors and industrial sensors.
 It supports [high cardinality data](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b),
@@ -219,7 +219,7 @@ perfectly [scales up on a single node](https://medium.com/@valyala/measuring-ver
 and scales horizontally to multiple nodes.
 
 
-### Where can I ask questions about VictoriaMetrics?
+## Where can I ask questions about VictoriaMetrics?
 
 Questions about VictoriaMetrics can be asked via the following channels:
 
@@ -228,17 +228,17 @@ Questions about VictoriaMetrics can be asked via the following channels:
 - [Google group](https://groups.google.com/forum/#!forum/victorametrics-users)
 
 
-### Where can I file bugs and feature requests regarding VictoriaMetrics?
+## Where can I file bugs and feature requests regarding VictoriaMetrics?
 
 File bugs and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
 
 
-### Where I can find information about multi-tenancy?
+## Where I can find information about multi-tenancy?
 
 See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy). Multitenancy is supported only by [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) of VictoriaMetrics.
 
 
-### How to set a memory limit for VictoriaMetrics components?
+## How to set a memory limit for VictoriaMetrics components?
 
 All the VictoriaMetrics component provide command-line flags to control the size of internal buffers and caches: `-memory.allowedPercent` and `-memory.allowedBytes` (pass `-help` to any VictoriaMetrics component in order to see the description for these flags). These limits don't take into account additional memory, which may be needed for processing incoming queries. Hard limits may be enforced only by the OS via [cgroups](https://en.wikipedia.org/wiki/Cgroups), Docker (see [these docs](https://docs.docker.com/config/containers/resource_constraints)) or Kubernetes (see [these docs](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers)).
 
@@ -250,22 +250,22 @@ Memory usage for VictoriaMetrics components can be tuned according to the follow
 * [Troubleshooting for single-node VictoriaMetrics](https://docs.victoriametrics.com/#troubleshooting)
 
 
-### How can I run VictoriaMetrics on FreeBSD?
+## How can I run VictoriaMetrics on FreeBSD?
 
 VictoriaMetrics is included in FreeBSD ports, so just install it from there. See [this link](https://www.freebsd.org/cgi/ports.cgi?query=victoria&stype=all).
 
 
-### Does VictoriaMetrics support Graphite query language?
+## Does VictoriaMetrics support Graphite query language?
 
 Yes. See [these docs](https://docs.victoriametrics.com/#graphite-api-usage).
 
 
-### What is active time series?
+## What is active time series?
 
 A time series is uniquely identified by its name plus a set of its labels. For example, `temperature{city="NY",country="US"}` and `temperature{city="SF",country="US"}` are two distinct series, since they differ by `city` label. A time series is considered active if it receives at least a single new sample during the last hour.
 
 
-### What is high churn rate?
+## What is high churn rate?
 
 If old time series are constantly substituted by new time series at a high rate, then such a state is called `high churn rate`. High churn rate has the following negative consequences:
 * Increased total number of time series stored in the database.
@@ -273,31 +273,31 @@ If old time series are constantly substituted by new time series at a high rate,
 * Slow down of queries over multiple days.
 
 
-### What is high cardinality?
+## What is high cardinality?
 
 High cardinality usually means high number of [active time series](#what-is-active-time-series). High cardinality may lead to high memory usage and/or to high percentage of [slow inserts](#what-is-slow-insert). The source of high cardinality is usually a label with big number of unique values, which presents in big share of the ingested time series. The solution is to identify and remove the source of high cardinality with the help of `/api/v1/status/tsdb` page - see [these docs](https://docs.victoriametrics.com/#tsdb-stats).
 
 
-### What is slow insert?
+## What is slow insert?
 
 VictoriaMetrics maintains in-memory cache for mapping of [active time series](#what-is-active-time-series) into internal series ids. The cache size depends on the available memory for VictoriaMetrics in the host system. If the information about all the active time series doesn't fit the cache, then VictoriaMetrics needs to read and unpack the information from disk on every incoming sample for time series missing in the cache. This operation is much slower than the cache lookup, so such insert is named `slow insert`. High percentage of slow inserts on the [official dashboard for VictoriaMetrics](https://docs.victoriametrics.com/#monitoring) indicates on memory shortage for the current number of [active time series](#what-is-active-time-series). Such a condition usually leads to significant slowdown for data ingestion, to significantly increased disk IO and CPU usage. The solution is to add more memory or to reduce the number of [active time series](#what-is-active-time-series). The `/api/v1/status/tsdb` page can be helpful for locating the source of high number of active time seriess - see [these docs](https://docs.victoriametrics.com/#tsdb-stats).
 
 
-### How to migrate data from Prometheus to VictoriaMetrics?
+## How to migrate data from Prometheus to VictoriaMetrics?
 
 Please see [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-prometheus).
 
 
-### How to migrate data from InfluxDB to VictoriaMetrics?
+## How to migrate data from InfluxDB to VictoriaMetrics?
 
 Please see [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-influxdb-1x).
 
 
-### How to migrate data from OpenTSDB to VictoriaMetrics?
+## How to migrate data from OpenTSDB to VictoriaMetrics?
 
 Please see [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-opentsdb).
 
 
-### How to migrate data from Graphite to VictoriaMetrics?
+## How to migrate data from Graphite to VictoriaMetrics?
 
 Please use [whisper-to-graphite](https://github.com/bzed/whisper-to-graphite) tool for reading the data from Graphite and pushing it to VictoriaMetrics via [Graphite import API](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index 711ac0ff2..97a5a7871 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -748,6 +748,10 @@ See also [implicit query conversions](#implicit-query-conversions).
 
 `limitk(k, q) by (group_labels)` returns up to `k` time series per each `group_labels` out of time series returned by `q`. The returned set of time series can change with each call.
 
+#### mad
+
+`mad(q) by (group_labels)` returns the [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) per each `group_labels` for all the time series returned by `q`. The aggregate is calculated individually per each group of points with the same timestamp. See also [outliers_mad](#outliers_mad) and [stddev](#stddev).
+
 #### max
 
 `max(q) by (group_labels)` returns the maximum value per each `group_labels` for all the time series returned by `q`. The aggregate is calculated individually per each group of points with the same timestamp. This function is supported by PromQL.
@@ -764,9 +768,13 @@ See also [implicit query conversions](#implicit-query-conversions).
 
 `mode(q) by (group_labels)` returns [mode](https://en.wikipedia.org/wiki/Mode_(statistics)) per each `group_labels` for all the time series returned by `q`. The aggregate is calculated individually per each group of points with the same timestamp.
 
+#### outliers_mad
+
+`outliers_mad(tolerance, q)` returns time series from `q` with at least a single point outside [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) (aka MAD) multiplied by `tolerance`. E.g. it returns time series with at least a single point below `median(q) - mad(q)` or a single point above `median(q) + mad(q)`. See also [outliersk](#outliersk) and [mad](#mad).
+
 #### outliersk
 
-`outliersk(k, q)` returns up to `k` time series with the biggest standard deviation (aka outliers) out of time series returned by `q`.
+`outliersk(k, q)` returns up to `k` time series with the biggest standard deviation (aka outliers) out of time series returned by `q`. See also [outliers_mad](#outliers_mad).
 
 #### quantile
 
diff --git a/go.mod b/go.mod
index 1bc2db3d4..9511e16ee 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.1.0
 	github.com/VictoriaMetrics/metrics v1.18.0
-	github.com/VictoriaMetrics/metricsql v0.21.0
+	github.com/VictoriaMetrics/metricsql v0.22.0
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aws/aws-sdk-go v1.40.43
 	github.com/cespare/xxhash/v2 v2.1.2
diff --git a/go.sum b/go.sum
index a9199a6b7..a4c0550ba 100644
--- a/go.sum
+++ b/go.sum
@@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ=
 github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
-github.com/VictoriaMetrics/metricsql v0.21.0 h1:wA/IVfRFQaThy4bM1kAmPiCR0BkWv4tEXD9lBF+GPdU=
-github.com/VictoriaMetrics/metricsql v0.21.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.22.0 h1:QWXvNfaFGeEQvh7Cjjpfp/7Un1N+W+ODQOiM+numyN0=
+github.com/VictoriaMetrics/metricsql v0.22.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go
index 27821de5f..7d6d7dceb 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go
@@ -35,6 +35,8 @@ var aggrFuncs = map[string]bool{
 	"bottomk_avg":    true,
 	"bottomk_median": true,
 	"any":            true,
+	"mad":            true,
+	"outliers_mad":   true,
 	"outliersk":      true,
 	"mode":           true,
 	"zscore":         true,
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
index 9bf873fc4..c5f0bfabd 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
@@ -134,7 +134,7 @@ func scanString(s string) (string, error) {
 	for {
 		n := strings.IndexByte(s[i:], quote)
 		if n < 0 {
-			return "", fmt.Errorf("cannot find closing quote %ch for the string %q", quote, s)
+			return "", fmt.Errorf("cannot find closing quote %c for the string %q", quote, s)
 		}
 		i += n
 		bs := 0
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 1726d8a2b..5e93f56ae 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -22,7 +22,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.18.0
 ## explicit
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.21.0
+# github.com/VictoriaMetrics/metricsql v0.22.0
 ## explicit
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop


Test patch:
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 25f2613d3..8c9f22cdd 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -5524,6 +5524,47 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`mad()`, func(t *testing.T) {
+		t.Parallel()
+		q := `mad(
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{100, 120, 140, 160, 180, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`outliers_mad(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `outliers_mad(1, (
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1500, 1800, 2100, 2400, 2700, 3000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("metric2")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`outliers_mad(5)`, func(t *testing.T) {
+		t.Parallel()
+		q := `outliers_mad(5, (
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`outliersk(0)`, func(t *testing.T) {
 		t.Parallel()
 		q := `outliersk(0, (
@@ -7000,6 +7041,9 @@ func TestExecError(t *testing.T) {
 	f(`hoeffding_bound_upper()`)
 	f(`hoeffding_bound_upper(1)`)
 	f(`hoeffding_bound_upper(0.99, foo, 1)`)
+	f(`mad()`)
+	f(`outliers_mad()`)
+	f(`outliers_mad(1)`)
 	f(`outliersk()`)
 	f(`outliersk(1)`)
 	f(`mode_over_time()`)