Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1505 (2021-07-30 11:03:54)

feature: Add multitenant for vmagent
This pull request adds the feature of handling multiple tenants by `vmagent`.
Example : If vmagent has received a write request on the accountID 2, It will remotewrite on the same tenant.

Main changes:
- Multiple tenant HTTP handling
- Dynamic creation of unique disk queue for each tenant
- Multiple tenant remotewrite using the tenant of input


Patch:
diff --git a/app/vmagent/csvimport/request_handler.go b/app/vmagent/csvimport/request_handler.go
index 3c5a70819..ac330972b 100644
--- a/app/vmagent/csvimport/request_handler.go
+++ b/app/vmagent/csvimport/request_handler.go
@@ -5,32 +5,36 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="csvimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
 )
 
 // InsertHandler processes csv data from req.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(p, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -64,8 +68,14 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(len(rows))
+		}
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }
diff --git a/app/vmagent/graphite/request_handler.go b/app/vmagent/graphite/request_handler.go
index fd0d96cd6..c3ef22d8d 100644
--- a/app/vmagent/graphite/request_handler.go
+++ b/app/vmagent/graphite/request_handler.go
@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
diff --git a/app/vmagent/influx/request_handler.go b/app/vmagent/influx/request_handler.go
index 9da1be933..a443af701 100644
--- a/app/vmagent/influx/request_handler.go
+++ b/app/vmagent/influx/request_handler.go
@@ -8,12 +8,15 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,8 +28,9 @@ var (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="influx"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
 )
 
 // InsertHandlerForReader processes remote write for influx line protocol.
@@ -35,7 +39,7 @@ var (
 func InsertHandlerForReader(r io.Reader) error {
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, nil)
+			return insertRows(nil, db, rows, nil)
 		})
 	})
 }
@@ -43,7 +47,7 @@ func InsertHandlerForReader(r io.Reader) error {
 // InsertHandlerForHTTP processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandlerForHTTP(req *http.Request) error {
+func InsertHandlerForHTTP(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -55,12 +59,12 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 		db := q.Get("db")
 		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, extraLabels)
+			return insertRows(p, db, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -130,8 +134,14 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.Push(&ctx.ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(rowsTotal)
+		}
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 
 	return nil
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index e1e28d5c1..2a3c7002c 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -19,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -88,7 +89,7 @@ func main() {
 
 	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.Init()
+	remotewrite.Init(nil)
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
@@ -159,11 +160,82 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		})
 		return true
 	}
+
+	p, err := httpserver.ParsePath(r.URL.Path)
+	if err == nil && p.Prefix == "insert" {
+
+		_, err := auth.NewToken(p.AuthToken)
+		if err != nil {
+			httpserver.Errorf(w, r, "auth error: %s", err)
+			return true
+		}
+
+		switch p.Suffix {
+		case "prometheus/", "prometheus", "prometheus/api/v1/write":
+			prometheusWriteRequests.Inc()
+			if err := promremotewrite.InsertHandler(p, r); err != nil {
+				prometheusWriteErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		case "prometheus/api/v1/import":
+			vmimportRequests.Inc()
+			if err := vmimport.InsertHandler(p, r); err != nil {
+				vmimportErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		case "prometheus/api/v1/import/csv":
+			csvimportRequests.Inc()
+			if err := csvimport.InsertHandler(p, r); err != nil {
+				csvimportErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		case "prometheus/api/v1/import/prometheus":
+			prometheusimportRequests.Inc()
+			if err := prometheusimport.InsertHandler(p, r); err != nil {
+				prometheusimportErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		case "prometheus/api/v1/import/native":
+			nativeimportRequests.Inc()
+			if err := native.InsertHandler(p, r); err != nil {
+				nativeimportErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		case "influx/write", "influx/api/v2/write":
+			influxWriteRequests.Inc()
+			if err := influx.InsertHandlerForHTTP(p, r); err != nil {
+				influxWriteErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			w.WriteHeader(http.StatusNoContent)
+			return true
+		default:
+			// This link is not multitenant
+		}
+
+	}
+
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := promremotewrite.InsertHandler(r); err != nil {
+		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -172,7 +244,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import":
 		vmimportRequests.Inc()
-		if err := vmimport.InsertHandler(r); err != nil {
+		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -181,7 +253,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
-		if err := csvimport.InsertHandler(r); err != nil {
+		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -190,7 +262,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/prometheus":
 		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(r); err != nil {
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
 			prometheusimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -199,7 +271,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/native":
 		nativeimportRequests.Inc()
-		if err := native.InsertHandler(r); err != nil {
+		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -208,7 +280,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandlerForHTTP(r); err != nil {
+		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
diff --git a/app/vmagent/native/request_handler.go b/app/vmagent/native/request_handler.go
index e93f5bb2b..b65af88a9 100644
--- a/app/vmagent/native/request_handler.go
+++ b/app/vmagent/native/request_handler.go
@@ -5,36 +5,40 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="native"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(block *parser.Block) error {
-			return insertRows(block, extraLabels)
+			return insertRows(p, block, extraLabels)
 		})
 	})
 }
 
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -42,6 +46,12 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	// since relabeling can prevent from inserting the rows.
 	rowsLen := len(block.Values)
 	rowsInserted.Add(rowsLen)
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(rowsLen)
+		}
+	}
 	rowsPerInsert.Update(float64(rowsLen))
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
@@ -80,6 +90,6 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.WriteRequest)
 	return nil
 }
diff --git a/app/vmagent/opentsdb/request_handler.go b/app/vmagent/opentsdb/request_handler.go
index 628676de9..2721912d6 100644
--- a/app/vmagent/opentsdb/request_handler.go
+++ b/app/vmagent/opentsdb/request_handler.go
@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
diff --git a/app/vmagent/opentsdbhttp/request_handler.go b/app/vmagent/opentsdbhttp/request_handler.go
index b3026ab86..7d2d409eb 100644
--- a/app/vmagent/opentsdbhttp/request_handler.go
+++ b/app/vmagent/opentsdbhttp/request_handler.go
@@ -65,7 +65,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
diff --git a/app/vmagent/prometheusimport/request_handler.go b/app/vmagent/prometheusimport/request_handler.go
index 45daa860b..c6cdf80a0 100644
--- a/app/vmagent/prometheusimport/request_handler.go
+++ b/app/vmagent/prometheusimport/request_handler.go
@@ -5,20 +5,24 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -30,12 +34,12 @@ func InsertHandler(req *http.Request) error {
 	return writeconcurrencylimiter.Do(func() error {
 		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(p, rows, extraLabels)
 		}, nil)
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -69,8 +73,14 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(len(rows))
+		}
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }
diff --git a/app/vmagent/promremotewrite/request_handler.go b/app/vmagent/promremotewrite/request_handler.go
index 0039e4ef4..64b3a91a7 100644
--- a/app/vmagent/promremotewrite/request_handler.go
+++ b/app/vmagent/promremotewrite/request_handler.go
@@ -5,34 +5,38 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="promremotewrite"}`)
 	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
-			return insertRows(tss, extraLabels)
+			return insertRows(p, tss, extraLabels)
 		})
 	})
 }
 
-func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -68,8 +72,14 @@ func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Labe
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(rowsTotal)
+		}
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }
diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index f2e87847b..5dcfc1805 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -42,11 +42,11 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.global = global
 	}
-	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), len(*remoteWriteURLs))
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.
@@ -58,6 +58,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.perURL[i] = prc
 	}
+
 	return &rcs, nil
 }
 
diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index 29a5ef1ff..70a294cc8 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -26,6 +27,9 @@ var (
 	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for remote storage URL to write data to. It must support VictoriaMetrics remote_write tenants API (identified by accountID or accountID:projectID). "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428 . "+
+		"Pass multiple -remoteWrite.multitenantURL flags in order to write data concurrently to multiple remote storage systems")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
@@ -53,7 +57,10 @@ var (
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -remoteWrite.maxHourlySeries")
 )
 
-var rwctxs []*remoteWriteCtx
+var defaultWriteToken = "default"
+var rwctxsMap = map[string][]*remoteWriteCtx{}
+
+var rwctxLock = sync.Mutex{}
 
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Value
@@ -75,10 +82,27 @@ func InitSecretFlags() {
 // It must be called after flag.Parse().
 //
 // Stop must be called for graceful shutdown.
-func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
+func Init(p *httpserver.Path) {
+	rwctxLock.Lock()
+	defer rwctxLock.Unlock()
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-remoteWrite.multitenantURL` command-line flag must be set")
+	}
+	// Do not Init MultitenantURLs they are dynamically initialized
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) > 0 && p == nil {
+		return
+	}
+
+	//  Create one writecontext per tenant
+	writeContextIndex := defaultWriteToken
+
+	if p != nil {
+		writeContextIndex = p.AuthToken
 	}
+	if _, ok := rwctxsMap[writeContextIndex]; ok {
+		return
+	}
+
 	if *maxHourlySeries > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
@@ -116,7 +140,7 @@ func Init() {
 	}
 	allRelabelConfigs.Store(rcs)
 
-	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
+	maxInmemoryBlocks := memory.Allowed() / (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) / maxRowsPerBlock / 100
 	if maxInmemoryBlocks > 400 {
 		// There is no much sense in keeping higher number of blocks in memory,
 		// since this means that the producer outperforms consumer and the queue
@@ -126,15 +150,34 @@ func Init() {
 	if maxInmemoryBlocks < 2 {
 		maxInmemoryBlocks = 2
 	}
-	for i, remoteWriteURL := range *remoteWriteURLs {
-		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
-		if *showRemoteWriteURL {
-			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+
+	rwctxs := []*remoteWriteCtx{}
+
+	if len(*remoteWriteURLs) > 0 && p == nil {
+		for i, remoteWriteURL := range *remoteWriteURLs {
+			sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+			if *showRemoteWriteURL {
+				sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+			}
+			rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+			rwctxs = append(rwctxs, rwctx)
+		}
+	}
+
+	if len(*remoteWriteMultitenantURLs) > 0 && p != nil {
+		for i, remoteWriteMultitenantURL := range *remoteWriteMultitenantURLs {
+			sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+			if *showRemoteWriteURL {
+				sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteMultitenantURL)
+			}
+			remoteWriteMultitenantURL := fmt.Sprintf("%s/%s/%s/%s", remoteWriteMultitenantURL, p.Prefix, p.AuthToken, p.Suffix)
+			rwctx := newRemoteWriteCtx(i, remoteWriteMultitenantURL, maxInmemoryBlocks, sanitizedURL)
+			rwctxs = append(rwctxs, rwctx)
 		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
-		rwctxs = append(rwctxs, rwctx)
 	}
 
+	rwctxsMap[writeContextIndex] = rwctxs
+
 	// Start config reloader.
 	configReloaderWG.Add(1)
 	go func() {
@@ -166,17 +209,32 @@ var configReloaderWG sync.WaitGroup
 func Stop() {
 	close(stopCh)
 	configReloaderWG.Wait()
-
-	for _, rwctx := range rwctxs {
-		rwctx.MustStop()
+	for _, rwctxs := range rwctxsMap {
+		for _, rwctx := range rwctxs {
+			rwctx.MustStop()
+		}
+		rwctxs = nil
 	}
-	rwctxs = nil
+	rwctxsMap = nil
 }
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
-func Push(wr *prompbmarshal.WriteRequest) {
+func Push(p *httpserver.Path, wr *prompbmarshal.WriteRequest) {
+	// if a queue is not created for this tenant, create it dynamically using the auth.Token
+	var rwctxs []*remoteWriteCtx
+	writeContextIndex := defaultWriteToken
+
+	// if no tenant speficied, p is nil
+	if p != nil {
+		writeContextIndex = p.AuthToken
+	}
+	if _, ok := rwctxsMap[writeContextIndex]; !ok {
+		Init(p)
+	}
+	rwctxs = rwctxsMap[writeContextIndex]
+
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
 	pcsGlobal := rcs.global
diff --git a/app/vmagent/vmimport/request_handler.go b/app/vmagent/vmimport/request_handler.go
index 5afb8a37c..b2fb10023 100644
--- a/app/vmagent/vmimport/request_handler.go
+++ b/app/vmagent/vmimport/request_handler.go
@@ -5,36 +5,40 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="vmimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(p *httpserver.Path, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(p, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(p *httpserver.Path, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -74,8 +78,14 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(p, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if p != nil {
+		at, err := auth.NewToken(p.AuthToken)
+		if err == nil {
+			rowsTenantInserted.Get(at).Add(rowsTotal)
+		}
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index faea8350d..7e1ae5fad 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -266,6 +266,8 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
 			logger.Errorf("cannot parse configuration file: %s", err)
 			continue
 		}
diff --git a/app/vmselect/main.go b/app/vmselect/main.go
index fd909b544..8a933b1d4 100644
--- a/app/vmselect/main.go
+++ b/app/vmselect/main.go
@@ -468,8 +468,8 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
 		return true
-	case "prometheus/api/v1/alerts":
-		// Return dumb placehloder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
+	case "prometheus/api/v1/alerts", "prometheus/alerts":
+		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
 		alertsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 377b7cc7a..97c353fcb 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -42,6 +42,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"stddev_over_time":   newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":   newRollupFuncOneArg(rollupStdvar),
 	"absent_over_time":   newRollupFuncOneArg(rollupAbsent),
+	"present_over_time":  newRollupFuncOneArg(rollupPresent),
 
 	// Additional rollup funcs.
 	"default_rollup":        newRollupFuncOneArg(rollupDefault), // default rollup func
@@ -97,23 +98,24 @@ var rollupFuncs = map[string]newRollupFunc{
 // rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
 var rollupAggrFuncs = map[string]rollupFunc{
 	// Standard rollup funcs from PromQL.
-	"changes":          rollupChanges,
-	"delta":            rollupDelta,
-	"deriv":            rollupDerivSlow,
-	"deriv_fast":       rollupDerivFast,
-	"idelta":           rollupIdelta,
-	"increase":         rollupDelta,     // + rollupFuncsRemoveCounterResets
-	"irate":            rollupIderiv,    // + rollupFuncsRemoveCounterResets
-	"rate":             rollupDerivFast, // + rollupFuncsRemoveCounterResets
-	"resets":           rollupResets,
-	"avg_over_time":    rollupAvg,
-	"min_over_time":    rollupMin,
-	"max_over_time":    rollupMax,
-	"sum_over_time":    rollupSum,
-	"count_over_time":  rollupCount,
-	"stddev_over_time": rollupStddev,
-	"stdvar_over_time": rollupStdvar,
-	"absent_over_time": rollupAbsent,
+	"changes":           rollupChanges,
+	"delta":             rollupDelta,
+	"deriv":             rollupDerivSlow,
+	"deriv_fast":        rollupDerivFast,
+	"idelta":            rollupIdelta,
+	"increase":          rollupDelta,     // + rollupFuncsRemoveCounterResets
+	"irate":             rollupIderiv,    // + rollupFuncsRemoveCounterResets
+	"rate":              rollupDerivFast, // + rollupFuncsRemoveCounterResets
+	"resets":            rollupResets,
+	"avg_over_time":     rollupAvg,
+	"min_over_time":     rollupMin,
+	"max_over_time":     rollupMax,
+	"sum_over_time":     rollupSum,
+	"count_over_time":   rollupCount,
+	"stddev_over_time":  rollupStddev,
+	"stdvar_over_time":  rollupStdvar,
+	"absent_over_time":  rollupAbsent,
+	"present_over_time": rollupPresent,
 
 	// Additional rollup funcs.
 	"range_over_time":     rollupRange,
@@ -157,6 +159,7 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{
 	"stddev_over_time":    true,
 	"stdvar_over_time":    true,
 	"absent_over_time":    true,
+	"present_over_time":   true,
 	"sum2_over_time":      true,
 	"geomean_over_time":   true,
 	"distinct_over_time":  true,
@@ -1284,6 +1287,15 @@ func rollupAbsent(rfa *rollupFuncArg) float64 {
 	return nan
 }
 
+func rollupPresent(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	if len(rfa.values) > 0 {
+		return 1
+	}
+	return nan
+}
+
 func rollupCount(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
diff --git a/app/vmui/packages/vmui/src/components/Home/Configurator/TimeSelector.tsx b/app/vmui/packages/vmui/src/components/Home/Configurator/TimeSelector.tsx
index 1a875c2e0..f8faaa4d0 100644
--- a/app/vmui/packages/vmui/src/components/Home/Configurator/TimeSelector.tsx
+++ b/app/vmui/packages/vmui/src/components/Home/Configurator/TimeSelector.tsx
@@ -71,7 +71,7 @@ export const TimeSelector: FC<TimeSelectorProps> = ({setDuration}) => {
             aria-haspopup="true"
             style={{cursor: "pointer"}}
             onMouseEnter={handlePopoverOpen}
-            onMouseLeave={handlePopoverClose}>�:&nbsp;</span>
+            onMouseLeave={handlePopoverClose}>:&nbsp;</span>
           <Popover
             open={open}
             anchorEl={anchorEl}
@@ -117,4 +117,4 @@ export const TimeSelector: FC<TimeSelectorProps> = ({setDuration}) => {
       </Box>
     </Box>
   </Box>;
-};
\ No newline at end of file
+};
diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml
index f8001151f..9011b5af2 100644
--- a/deployment/docker/alerts.yml
+++ b/deployment/docker/alerts.yml
@@ -18,11 +18,31 @@ groups:
         expr: up{job=~"vmselect|vminsert|vmstorage|vmagent|vmalert"} == 0
         for: 2m
         labels:
-          severity: "critical"
+          severity: critical
         annotations:
           summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
           description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."
 
+      - alert: ProcessNearFDLimits
+        expr: (process_max_fds - process_open_fds) < 100
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
+          description: "Exhausting OS file descriptors limit can cause severe degradation of the process.
+          Consider to increase the limit as fast as possible."
+
+      - alert: TooHighMemoryUsage
+        expr: (process_resident_memory_anon_bytes / vm_available_memory_bytes) > 0.9
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "It is more than 90% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
+          description: "Too high memory usage may result into multiple issues such as OOMs or degraded performance.
+           Consider to either increase available memory or decrease the load on the process."
+
   # Alerts group for VM cluster assumes that Grafana dashboard
   # https://grafana.com/grafana/dashboards/11176 is installed.
   # Please, update the `dashboard` annotation according to your setup.
diff --git a/docs/Articles.md b/docs/Articles.md
index ccf964c37..a96f3d1ae 100644
--- a/docs/Articles.md
+++ b/docs/Articles.md
@@ -6,19 +6,20 @@ sort: 16
 
 ## Third-party articles and slides about VictoriaMetrics
 
+* [Scaling to trillions of metric data points](https://engineering.razorpay.com/scaling-to-trillions-of-metric-data-points-f569a5b654f2)
 * [VictoriaMetrics vs. OpenTSDB](https://blg.robot-house.us/posts/tsdbs-grow/)
 * [Monitoring of multiple OpenShift clusters with VictoriaMetrics](https://medium.com/ibm-garage/monitoring-of-multiple-openshift-clusters-with-victoriametrics-d4f0979e2544)
 * [Fly's Prometheus Metrics](https://fly.io/blog/measuring-fly/)
+* [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
+* [Sismology: Iguana Solutions’ Monitoring System](https://medium.com/nerd-for-tech/sismology-iguana-solutions-monitoring-system-f46e4170447f)
+* [Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
+* [How we improved our Kubernetes monitoring at Smarkets, and how you could too](https://smarketshq.com/monitoring-kubernetes-clusters-41a4b24c19e3)
 * [Foiled by the Firewall: A Tale of Transition From Prometheus to VictoriaMetrics](https://www.percona.com/blog/2020/12/01/foiled-by-the-firewall-a-tale-of-transition-from-prometheus-to-victoriametrics/)
 * [Observations on Better Resource Usage with Percona Monitoring and Management v2.12.0](https://www.percona.com/blog/2020/12/23/observations-on-better-resource-usage-with-percona-monitoring-and-management-v2-12-0/)
 * [Better Prometheus rate() function with VictoriaMetrics](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/)
 * [Percona monitoring and management migration from Prometheus to VictoriaMetrics FAQ](https://www.percona.com/blog/2020/12/16/percona-monitoring-and-management-migration-from-prometheus-to-victoriametrics-faq/)
 * [Compiling a Percona Monitoring and Management v2 Client in ARM: Raspberry Pi 3 Reprise](https://www.percona.com/blog/2021/05/26/compiling-a-percona-monitoring-and-management-v2-client-in-arm-raspberry-pi-3/)
 * [Making peace with Prometheus rate()](https://blog.doit-intl.com/making-peace-with-prometheus-rate-43a3ea75c4cf)
-* [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
-* [Sismology: Iguana Solutions’ Monitoring System](https://medium.com/@IG1.com/sismology-iguana-solutions-monitoring-system-f46e4170447f)
-* [Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
-* [How we improved our Kubernetes monitoring at Smarkets, and how you could too](https://smarketshq.com/monitoring-kubernetes-clusters-41a4b24c19e3)
 * [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit)
 * [CMS monitoring R&D: Real-time monitoring and alerts](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
 * [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
@@ -40,54 +41,55 @@ sort: 16
 * [Monitoring Kubernetes clusters with VictoriaMetrics and Grafana](https://blog.cybozu.io/entry/2021/03/18/115743)
 * [Multi-tenancy monitoring system for Kubernetes cluster using VictoriaMetrics and operators](https://blog.kintone.io/entry/2021/03/31/175256)
 * [Monitoring as Code на базе VictoriaMetrics и Grafana](https://habr.com/ru/post/568090/)
+* [Push Prometheus metrics to VictoriaMetrics or other exporters](https://pythonawesome.com/push-prometheus-metrics-to-victoriametrics-or-other-exporters/)
 
 
 ## Our articles
 
 ### Announcements
 
-* [Open-sourcing VictoriaMetrics](https://medium.com/@valyala/open-sourcing-victoriametrics-f31e34485c2b)
-* [How we created VictoriaMetrics](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac)
-* [Anomaly Detection in VictoriaMetrics](https://medium.com/@VictoriaMetrics/anomaly-detection-in-victoriametrics-9528538786a7)
+* [Open-sourcing VictoriaMetrics](https://blog.usejournal.com/open-sourcing-victoriametrics-f31e34485c2b)
+* [VictoriaMetrics — creating the best remote storage for Prometheus](https://faun.pub/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac)
+* [Anomaly Detection in VictoriaMetrics](https://victoriametrics.medium.com/anomaly-detection-in-victoriametrics-9528538786a7)
 
 
 ### Benchmarks
 
-* [VictoriaMetrics vs TimescaleDB vs InfluxDB benchmarks on 40K unique time series](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
-* [VictoriaMetrics vs TimescaleDB vs InfluxDB benchmarks on 400K, 4M and 40M unique time series](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
-* [Insert benchmarks for VictoriaMetrics vs InfluxDB on high-cardinality data](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
-* [Measuring vertical scalability for time series databases in Google Cloud](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
-* [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://medium.com/@valyala/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)
-* [First look at performance comparison between InfluxDB IOx and VictoriaMetrics](https://medium.com/@VictoriaMetrics/first-look-at-perfomance-comparassion-between-influxdb-iox-and-victoriametrics-e590f847935b)
+* [When size matters — benchmarking VictoriaMetrics vs Timescale and InfluxDB](https://valyala.medium.com/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+* [High-cardinality TSDB benchmarks: VictoriaMetrics vs TimescaleDB vs InfluxDB](https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+* [Insert benchmarks with inch: InfluxDB vs VictoriaMetrics](https://valyala.medium.com/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
+* [Measuring vertical scalability for time series databases in Google Cloud](https://valyala.medium.com/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+* [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://valyala.medium.com/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)
+* [First look at performance comparison between InfluxDB IOx and VictoriaMetrics](https://victoriametrics.medium.com/first-look-at-perfomance-comparassion-between-influxdb-iox-and-victoriametrics-e590f847935b)
 * [Prometheus vs VictoriaMetrics benchmark on node-exporter metrics](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f)
 * [Promscale vs VictoriaMetrics: resource usage on production workload](https://valyala.medium.com/promscale-vs-victoriametrics-resource-usage-on-production-workload-91c8e3786c03)
 
 
 ### Technical articles
 
-* [How VictoriaMetrics creates instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
-* [WAL Usage Looks Broken in Modern TSDBs](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704)
-* [Why mmap'ed files in Go may hurt performance](https://medium.com/@valyala/mmap-in-go-considered-harmful-d92a25cb161d)
-* [Achieving better compression for time series data than Gorilla](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
-* [Stripping dependency bloat in VictoriaMetrics Docker image](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
-* [Speeding up backups for big time series databases](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883)
-* [Improving histogram usability for Prometheus and Grafana](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
-* [Why irate from Prometheus doesn't capture spikes](https://medium.com/@valyala/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
+* [How VictoriaMetrics makes instant snapshots for multi-terabyte time series data](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+* [WAL Usage Looks Broken in Modern TSDBs](https://valyala.medium.com/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704)
+* [mmap may slow down your Go app](https://valyala.medium.com/mmap-in-go-considered-harmful-d92a25cb161d)
+* [VictoriaMetrics: achieving better compression than Gorilla for time series data](https://faun.pub/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
+* [Stripping dependency bloat in VictoriaMetrics Docker image](https://valyala.medium.com/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
+* [Speeding up backups for big time series databases](https://valyala.medium.com/speeding-up-backups-for-big-time-series-databases-533c1a927883)
+* [Improving histogram usability for Prometheus and Grafana](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
+* [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
 
 
 ### Tutorials, guides and how-to articles
 
-* [PromQL tutorial for beginners](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085)
-* [Analyzing Prometheus data with external tools](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639)
-* [Prometheus Subqueries in VictoriaMetrics](https://medium.com/@valyala/prometheus-subqueries-in-victoriametrics-9b1492b720b3)
+* [PromQL tutorial for beginners and humans](https://valyala.medium.com/promql-tutorial-for-beginners-9ab455142085)
+* [Analyzing Prometheus data with external tools](https://valyala.medium.com/analyzing-prometheus-data-with-external-tools-5f3e5e147639)
+* [Prometheus Subqueries in VictoriaMetrics](https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3)
 * [How to migrate data from Prometheus to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
-* [Filtering and modifying time series during import to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
+* [VictoriaMetrics: how to migrate data from Prometheus. Filtering and modifying time series.](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
 * [How to use relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
 * [How to monitor Go applications with VictoriaMetrics](https://victoriametrics.medium.com/how-to-monitor-go-applications-with-victoriametrics-c04703110870)
-* [Prometheus storage: tech terms for humans](https://medium.com/@valyala/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
+* [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
 
 
 ### Other articles
 
-* [Comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
-* [Evaluation performance and correctness: VictoriaMetrics response](https://medium.com/@valyala/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)
+* [Comparing Thanos to VictoriaMetrics cluster](https://faun.pub/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+* [Evaluation performance and correctness: VictoriaMetrics response](https://valyala.medium.com/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index bbb91ef25..200992b7a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -6,6 +6,9 @@ sort: 15
 
 ## tip
 
+* FEATURE: add `present_over_time(m[d])` function, which returns 1 if `m` has a least a single sample over the previous duration `d`. This function has been added also to [Prometheus 2.29](https://github.com/prometheus/prometheus/releases/tag/v2.29.0-rc.0).
+* FEATURE: vmagent: add `__meta_ec2_availability_zone_id` label to discovered Amazon EC2 targets. This label is available in Prometheus [starting from v2.29](https://github.com/prometheus/prometheus/releases/tag/v2.29.0-rc.0).
+* FAETURE: vmagent: add `__meta_gce_interface_ipv4_<name>` labels to discovered GCE targets. These labels are available in Prometheus [starting from v2.29](https://github.com/prometheus/prometheus/releases/tag/v2.29.0-rc.0).
 * FEATURE: add `-search.maxSamplesPerSeries` command-line flag for limiting the number of raw samples a single query can process per each time series. This option can protect from out of memory errors when a query processes tens of millions of raw samples per series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1067).
 * FEATURE: add `-search.maxSamplesPerQuery` command-line flag for limiting the number of raw samples a single query can process across all the time series. This option can protect from heavy queries, which select too big number of raw samples. Thanks to @jiangxinlingdu for [the initial pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1478).
 * FEATURE: improve performance for queries that process big number of time series and/or samples on systems with big number of CPU cores.
diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md
index 777f0876c..589de0605 100644
--- a/docs/CaseStudies.md
+++ b/docs/CaseStudies.md
@@ -108,7 +108,7 @@ After using Cacti, Graphite and StatsD for years, we wanted to upgrade our monit
 - is scalable
 - has a simple client that allows for provisioning and discovery with Puppet
 
-We hed been running Prometheus for about a year in a test environment and it was working well but there was a need/wish for a few more years of retention than the old system provided. We tested Thanos which was a bit resource hungry but worked great for about half a year.
+We had been running Prometheus for about a year in a test environment and it was working well but there was a need/wish for a few more years of retention than the old system provided. We tested Thanos which was a bit resource hungry but worked great for about half a year.
 Then we discovered VictoriaMetrics. Our scale isn't that big so we don't have on-prem S3 and no Kubernetes. VM's single node instance provided
 the same result with far less maintenance overhead and lower hardware requirements.
 
diff --git a/docs/guides/k8s-monitoring-via-vm-cluster.md b/docs/guides/k8s-monitoring-via-vm-cluster.md
index a05a50801..2b7de271b 100644
--- a/docs/guides/k8s-monitoring-via-vm-cluster.md
+++ b/docs/guides/k8s-monitoring-via-vm-cluster.md
@@ -20,7 +20,7 @@ We will use:
   <img src="guide-vmcluster-k8s-scheme.png" width="800" alt="VictoriaMetrics Cluster on Kubernetes cluster">
 </p>
 
-### 1. VictoriaMetrics Helm repository
+## 1. VictoriaMetrics Helm repository
 
 > For this guide we will use Helm 3 but if you already use Helm 2 please see this [https://github.com/VictoriaMetrics/helm-charts#for-helm-v2](https://github.com/VictoriaMetrics/helm-charts#for-helm-v2)
 
@@ -67,7 +67,7 @@ vm/victoria-metrics-operator 	0.1.17       	0.16.0     	Victoria Metrics Operato
 vm/victoria-metrics-single   	0.7.5        	1.62.0     	Victoria Metrics Single version - high-performa...
 ```
 
-### 2. Install VictoriaMetrics Cluster from the Helm chart
+## 2. Install VictoriaMetrics Cluster from the Helm chart
 
 Run this command in your terminal:
 
@@ -176,7 +176,7 @@ vmcluster-victoria-metrics-cluster-vmstorage-0                 1/1     Running
 vmcluster-victoria-metrics-cluster-vmstorage-1                 1/1     Running   0          16m
 ```
 
-### 3. Install vmagent from the Helm chart
+## 3. Install vmagent from the Helm chart
 
 To scrape metrics from Kubernetes with a [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) we need to install [vmagent](https://docs.victoriametrics.com/vmagent.html) with additional configuration. To do so, please run these commands in your terminal:
 
@@ -434,7 +434,7 @@ vmagent-victoria-metrics-agent-69974b95b4-mhjph                1/1     Running
 ```
 
 
-### 4. Install and connect Grafana to VictoriaMetrics with Helm
+## 4. Install and connect Grafana to VictoriaMetrics with Helm
 
 Add the Grafana Helm repository. 
 
@@ -521,7 +521,7 @@ kubectl --namespace default port-forward $POD_NAME 3000
 ```
 </div>
 
-### 5. Check the result you obtained in your browser
+## 5. Check the result you obtained in your browser
 
 To check that [VictoriaMetrics](https://victoriametrics.com) collects metrics from k8s cluster open in browser [http://127.0.0.1:3000/dashboards](http://127.0.0.1:3000/dashboards) and choose the `Kubernetes Cluster Monitoring (via Prometheus)` dashboard. Use `admin` for login and `password` that you previously got from kubectl. 
 
@@ -544,7 +544,7 @@ vmagent has it’s own dashboard:
   <img src="guide-vmcluster-vmagent-grafana-dash.png" width="800" alt="vmagent dashboard">
 </p>
 
-### 6. Final thoughts
+## 6. Final thoughts
 
 * We set up TimeSeries Database for your Kubernetes cluster.
 * We collected metrics from all running pods,nodes, … and stored them in a VictoriaMetrics database.
diff --git a/docs/guides/k8s-monitoring-via-vm-single.md b/docs/guides/k8s-monitoring-via-vm-single.md
index b9d680d33..afe0c8659 100644
--- a/docs/guides/k8s-monitoring-via-vm-single.md
+++ b/docs/guides/k8s-monitoring-via-vm-single.md
@@ -20,7 +20,7 @@ We will use:
   <img src="guide-vmsingle-k8s-scheme.png" width="800" alt="VictoriaMetrics Single on Kubernetes cluster">
 </p>
 
-### 1. VictoriaMetrics Helm repository
+## 1. VictoriaMetrics Helm repository
 
 > For this guide we will use Helm 3 but if you already use Helm 2 please see this [https://github.com/VictoriaMetrics/helm-charts#for-helm-v2](https://github.com/VictoriaMetrics/helm-charts#for-helm-v2)
 
@@ -68,7 +68,7 @@ vm/victoria-metrics-single   	0.7.5        	1.62.0     	Victoria Metrics Single
 ```
 
 
-### 2. Install [VictoriaMetrics Single](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) from Helm Chart
+## 2. Install [VictoriaMetrics Single](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) from Helm Chart
 
 Run this command in your terminal:
 
@@ -233,7 +233,7 @@ vmsingle-victoria-metrics-single-server-0   1/1     Running   0          68s
 ```
 
 
-### 3. Install and connect Grafana to VictoriaMetrics with Helm
+## 3. Install and connect Grafana to VictoriaMetrics with Helm
 
 Add the Grafana Helm repository. 
 
@@ -326,7 +326,7 @@ kubectl --namespace default port-forward $POD_NAME 3000
 Now Grafana should be accessible on the [http://127.0.0.1:3000](http://127.0.0.1:3000) address.
 
 
-### 4. Check the obtained result in your browser
+## 4. Check the obtained result in your browser
 
 To check that VictoriaMetrics has collects metrics from the k8s cluster open in browser [http://127.0.0.1:3000/dashboards](http://127.0.0.1:3000/dashboards) and choose `Kubernetes Cluster Monitoring (via Prometheus)` dashboard. Use `admin` for login and `password` that you previously obtained from kubectl. 
 
@@ -344,7 +344,7 @@ VictoriaMetrics dashboard also available to use:
   <img src="guide-vmsingle-grafana.png" width="800" alt="">
 </p>
 
-### 5. Final thoughts
+## 5. Final thoughts
 
 * We have set up TimeSeries Database for your k8s cluster.
 * Collected metrics from all running pods,nodes, … and store them in VictoriaMetrics database.
diff --git a/go.mod b/go.mod
index 0a9f29a4a..0031faeb3 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.16
 	github.com/VictoriaMetrics/metrics v1.17.3
-	github.com/VictoriaMetrics/metricsql v0.16.0
+	github.com/VictoriaMetrics/metricsql v0.17.0
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aws/aws-sdk-go v1.40.7
 	github.com/cespare/xxhash/v2 v2.1.1
diff --git a/go.sum b/go.sum
index 1adf9e59c..dd291aa84 100644
--- a/go.sum
+++ b/go.sum
@@ -107,8 +107,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4=
 github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.16.0 h1:YzrMnGUs6Y6f5LxsH8eSAoik98aEzlc1TiYgOONgr3Q=
-github.com/VictoriaMetrics/metricsql v0.16.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.17.0 h1:OdOkx5GK2p1+EnHn/+zpLWW6ze96L1AmMV5X1HOzyfY=
+github.com/VictoriaMetrics/metricsql v0.17.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go
index 9a13dcb9d..e72c70223 100644
--- a/lib/promscrape/discovery/ec2/api.go
+++ b/lib/promscrape/discovery/ec2/api.go
@@ -33,6 +33,10 @@ type apiConfig struct {
 	// Real credentials used for accessing EC2 API.
 	creds     *apiCredentials
 	credsLock sync.Mutex
+
+	// A map from AZ name to AZ id.
+	azMap     map[string]string
+	azMapLock sync.Mutex
 }
 
 // apiCredentials represents aws api credentials
diff --git a/lib/promscrape/discovery/ec2/instance.go b/lib/promscrape/discovery/ec2/instance.go
index 45ccfb0cf..37425e2d2 100644
--- a/lib/promscrape/discovery/ec2/instance.go
+++ b/lib/promscrape/discovery/ec2/instance.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -14,10 +15,11 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
 	if err != nil {
 		return nil, err
 	}
+	azMap := getAZMap(cfg)
 	var ms []map[string]string
 	for _, r := range rs {
 		for _, inst := range r.InstanceSet.Items {
-			ms = inst.appendTargetLabels(ms, r.OwnerID, cfg.port)
+			ms = inst.appendTargetLabels(ms, r.OwnerID, cfg.port, azMap)
 		}
 	}
 	return ms, nil
@@ -25,11 +27,10 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
 
 func getReservations(cfg *apiConfig) ([]Reservation, error) {
 	// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
-	action := "DescribeInstances"
 	var rs []Reservation
 	pageToken := ""
 	for {
-		data, err := getEC2APIResponse(cfg, action, pageToken)
+		data, err := getEC2APIResponse(cfg, "DescribeInstances", pageToken)
 		if err != nil {
 			return nil, fmt.Errorf("cannot obtain instances: %w", err)
 		}
@@ -132,29 +133,87 @@ func parseInstancesResponse(data []byte) (*InstancesResponse, error) {
 	return &v, nil
 }
 
-func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string, port int) []map[string]string {
+func getAZMap(cfg *apiConfig) map[string]string {
+	cfg.azMapLock.Lock()
+	defer cfg.azMapLock.Unlock()
+
+	if cfg.azMap != nil {
+		return cfg.azMap
+	}
+
+	azs, err := getAvailabilityZones(cfg)
+	cfg.azMap = make(map[string]string, len(azs))
+	if err != nil {
+		logger.Warnf("couldn't load availability zones map, so __meta_ec2_availability_zone_id label isn't set: %s", err)
+		return cfg.azMap
+	}
+	for _, az := range azs {
+		cfg.azMap[az.ZoneName] = az.ZoneID
+	}
+	return cfg.azMap
+}
+
+func getAvailabilityZones(cfg *apiConfig) ([]AvailabilityZone, error) {
+	// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
+	data, err := getEC2APIResponse(cfg, "DescribeAvailabilityZones", "")
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain availability zones: %w", err)
+	}
+	azr, err := parseAvailabilityZonesResponse(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse availability zones list: %w", err)
+	}
+	return azr.AvailabilityZoneInfo.Items, nil
+}
+
+// AvailabilityZonesResponse represents the response for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
+type AvailabilityZonesResponse struct {
+	AvailabilityZoneInfo AvailabilityZoneInfo `xml:"availabilityZoneInfo"`
+}
+
+// AvailabilityZoneInfo represents availabilityZoneInfo for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
+type AvailabilityZoneInfo struct {
+	Items []AvailabilityZone `xml:"item"`
+}
+
+// AvailabilityZone represents availabilityZone for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_AvailabilityZone.html
+type AvailabilityZone struct {
+	ZoneName string `xml:"zoneName"`
+	ZoneID   string `xml:"zoneId"`
+}
+
+func parseAvailabilityZonesResponse(data []byte) (*AvailabilityZonesResponse, error) {
+	var v AvailabilityZonesResponse
+	if err := xml.Unmarshal(data, &v); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal DescribeAvailabilityZonesResponse from %q: %w", data, err)
+	}
+	return &v, nil
+}
+
+func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string, port int, azMap map[string]string) []map[string]string {
 	if len(inst.PrivateIPAddress) == 0 {
 		// Cannot scrape instance without private IP address
 		return ms
 	}
 	addr := discoveryutils.JoinHostPort(inst.PrivateIPAddress, port)
 	m := map[string]string{
-		"__address__":                   addr,
-		"__meta_ec2_architecture":       inst.Architecture,
-		"__meta_ec2_ami":                inst.ImageID,
-		"__meta_ec2_availability_zone":  inst.Placement.AvailabilityZone,
-		"__meta_ec2_instance_id":        inst.ID,
-		"__meta_ec2_instance_lifecycle": inst.Lifecycle,
-		"__meta_ec2_instance_state":     inst.State.Name,
-		"__meta_ec2_instance_type":      inst.Type,
-		"__meta_ec2_owner_id":           ownerID,
-		"__meta_ec2_platform":           inst.Platform,
-		"__meta_ec2_primary_subnet_id":  inst.SubnetID,
-		"__meta_ec2_private_dns_name":   inst.PrivateDNSName,
-		"__meta_ec2_private_ip":         inst.PrivateIPAddress,
-		"__meta_ec2_public_dns_name":    inst.PublicDNSName,
-		"__meta_ec2_public_ip":          inst.PublicIPAddress,
-		"__meta_ec2_vpc_id":             inst.VPCID,
+		"__address__":                     addr,
+		"__meta_ec2_architecture":         inst.Architecture,
+		"__meta_ec2_ami":                  inst.ImageID,
+		"__meta_ec2_availability_zone":    inst.Placement.AvailabilityZone,
+		"__meta_ec2_availability_zone_id": azMap[inst.Placement.AvailabilityZone],
+		"__meta_ec2_instance_id":          inst.ID,
+		"__meta_ec2_instance_lifecycle":   inst.Lifecycle,
+		"__meta_ec2_instance_state":       inst.State.Name,
+		"__meta_ec2_instance_type":        inst.Type,
+		"__meta_ec2_owner_id":             ownerID,
+		"__meta_ec2_platform":             inst.Platform,
+		"__meta_ec2_primary_subnet_id":    inst.SubnetID,
+		"__meta_ec2_private_dns_name":     inst.PrivateDNSName,
+		"__meta_ec2_private_ip":           inst.PrivateIPAddress,
+		"__meta_ec2_public_dns_name":      inst.PublicDNSName,
+		"__meta_ec2_public_ip":            inst.PublicIPAddress,
+		"__meta_ec2_vpc_id":               inst.VPCID,
 	}
 	if len(inst.VPCID) > 0 {
 		subnets := make([]string, 0, len(inst.NetworkInterfaceSet.Items))
diff --git a/lib/promscrape/discovery/gce/instance.go b/lib/promscrape/discovery/gce/instance.go
index 345f7189c..65ad7bdd7 100644
--- a/lib/promscrape/discovery/gce/instance.go
+++ b/lib/promscrape/discovery/gce/instance.go
@@ -93,6 +93,7 @@ type Instance struct {
 
 // NetworkInterface is network interface from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list
 type NetworkInterface struct {
+	Name          string
 	Network       string
 	Subnetwork    string
 	NetworkIP     string
@@ -148,6 +149,10 @@ func (inst *Instance) appendTargetLabels(ms []map[string]string, project, tagSep
 		"__meta_gce_subnetwork":      iface.Subnetwork,
 		"__meta_gce_zone":            inst.Zone,
 	}
+	for _, iface := range inst.NetworkInterfaces {
+		ifaceName := discoveryutils.SanitizeLabelName(iface.Name)
+		m["__meta_gce_interface_ipv4_"+ifaceName] = iface.NetworkIP
+	}
 	if len(inst.Tags.Items) > 0 {
 		// We surround the separated list with the separator as well. This way regular expressions
 		// in relabeling rules don't have to consider tag positions.
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 8509636d6..6b661566d 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -8,6 +8,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -49,7 +50,7 @@ func CheckConfig() error {
 // Init initializes Prometheus scraper with config from the `-promscrape.config`.
 //
 // Scraped data is passed to pushData.
-func Init(pushData func(wr *prompbmarshal.WriteRequest)) {
+func Init(pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)) {
 	globalStopCh = make(chan struct{})
 	scraperWG.Add(1)
 	go func() {
@@ -72,7 +73,7 @@ var (
 	PendingScrapeConfigs int32
 )
 
-func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest), globalStopCh <-chan struct{}) {
+func runScraper(configFile string, pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest), globalStopCh <-chan struct{}) {
 	if configFile == "" {
 		// Nothing to scrape.
 		return
@@ -160,13 +161,13 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 var configReloads = metrics.NewCounter(`vm_promscrape_config_reloads_total`)
 
 type scrapeConfigs struct {
-	pushData func(wr *prompbmarshal.WriteRequest)
+	pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)
 	wg       sync.WaitGroup
 	stopCh   chan struct{}
 	scfgs    []*scrapeConfig
 }
 
-func newScrapeConfigs(pushData func(wr *prompbmarshal.WriteRequest)) *scrapeConfigs {
+func newScrapeConfigs(pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)) *scrapeConfigs {
 	return &scrapeConfigs{
 		pushData: pushData,
 		stopCh:   make(chan struct{}),
@@ -207,7 +208,7 @@ func (scs *scrapeConfigs) stop() {
 
 type scrapeConfig struct {
 	name          string
-	pushData      func(wr *prompbmarshal.WriteRequest)
+	pushData      func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)
 	getScrapeWork func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork
 	checkInterval time.Duration
 	cfgCh         chan *Config
@@ -256,7 +257,7 @@ type scraperGroup struct {
 	wg       sync.WaitGroup
 	mLock    sync.Mutex
 	m        map[string]*scraper
-	pushData func(wr *prompbmarshal.WriteRequest)
+	pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)
 
 	changesCount    *metrics.Counter
 	activeScrapers  *metrics.Counter
@@ -264,7 +265,7 @@ type scraperGroup struct {
 	scrapersStopped *metrics.Counter
 }
 
-func newScraperGroup(name string, pushData func(wr *prompbmarshal.WriteRequest)) *scraperGroup {
+func newScraperGroup(name string, pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)) *scraperGroup {
 	sg := &scraperGroup{
 		name:     name,
 		m:        make(map[string]*scraper),
@@ -358,7 +359,7 @@ type scraper struct {
 	stopCh chan struct{}
 }
 
-func newScraper(sw *ScrapeWork, group string, pushData func(wr *prompbmarshal.WriteRequest)) *scraper {
+func newScraper(sw *ScrapeWork, group string, pushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)) *scraper {
 	sc := &scraper{
 		stopCh: make(chan struct{}),
 	}
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 281a0e57e..90cd3b9d3 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/leveledbytebufferpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -163,7 +164,7 @@ type scrapeWork struct {
 	GetStreamReader func() (*streamReader, error)
 
 	// PushData is called for pushing collected data.
-	PushData func(wr *prompbmarshal.WriteRequest)
+	PushData func(p *httpserver.Path, wr *prompbmarshal.WriteRequest)
 
 	// ScrapeGroup is name of ScrapeGroup that
 	// scrapeWork belongs to
@@ -316,7 +317,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
 	sw.addAutoTimeseries(wc, "scrape_samples_post_metric_relabeling", float64(samplesPostRelabeling), scrapeTimestamp)
 	sw.addAutoTimeseries(wc, "scrape_series_added", float64(seriesAdded), scrapeTimestamp)
 	startTime := time.Now()
-	sw.PushData(&wc.writeRequest)
+	sw.PushData(nil, &wc.writeRequest)
 	pushDataDuration.UpdateDuration(startTime)
 	sw.prevLabelsLen = len(wc.labels)
 	wc.reset()
@@ -358,7 +359,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 			}
 			sw.updateSeriesAdded(wc)
 			startTime := time.Now()
-			sw.PushData(&wc.writeRequest)
+			sw.PushData(nil, &wc.writeRequest)
 			pushDataDuration.UpdateDuration(startTime)
 			wc.resetNoRows()
 			return nil
@@ -386,7 +387,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	sw.addAutoTimeseries(wc, "scrape_samples_post_metric_relabeling", float64(samplesPostRelabeling), scrapeTimestamp)
 	sw.addAutoTimeseries(wc, "scrape_series_added", float64(seriesAdded), scrapeTimestamp)
 	startTime := time.Now()
-	sw.PushData(&wc.writeRequest)
+	sw.PushData(nil, &wc.writeRequest)
 	pushDataDuration.UpdateDuration(startTime)
 	sw.prevLabelsLen = len(wc.labels)
 	wc.reset()
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
index 085ee96c7..4db4770f4 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
@@ -27,6 +27,7 @@ var rollupFuncs = map[string]bool{
 	"stddev_over_time":   true,
 	"stdvar_over_time":   true,
 	"absent_over_time":   true,
+	"present_over_time":  true,
 
 	// Additional rollup funcs.
 	"default_rollup":        true,
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 6dc29ce19..f0226bf9f 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.17.3
 ## explicit
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.16.0
+# github.com/VictoriaMetrics/metricsql v0.17.0
 ## explicit
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop


Test patch:
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index d687f50a4..7a07fa56c 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -708,6 +708,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run("present_over_time(time())", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("present_over_time(time()[100:300])", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time()[100:300])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1, nan, nan, 1, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("present_over_time(time()<10m)", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time()<1600)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("absent(123)", func(t *testing.T) {
 		t.Parallel()
 		q := `absent(123)`
diff --git a/lib/promscrape/discovery/ec2/instance_test.go b/lib/promscrape/discovery/ec2/instance_test.go
index a19329264..a8441a96c 100644
--- a/lib/promscrape/discovery/ec2/instance_test.go
+++ b/lib/promscrape/discovery/ec2/instance_test.go
@@ -8,6 +8,56 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
+func TestDescribeAvailabilityZonesResponse(t *testing.T) {
+	data := `<DescribeAvailabilityZonesResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>e23c5a54-a29c-43ee-8b55-0c13c26e9e01</requestId>
+    <availabilityZoneInfo>
+        <item>
+            <optInStatus>opt-in-not-required</optInStatus>
+            <zoneName>us-west-2a</zoneName>
+            <zoneId>usw2-az1</zoneId>
+            <zoneState>available</zoneState>
+            <regionName>us-west-2</regionName>
+            <messageSet/>
+            <NetworkBorderGroup>us-west-2-lax-1</NetworkBorderGroup>
+        </item>
+        <item>
+            <groupName>us-west-2</groupName>
+            <optInStatus>opt-in-not-required</optInStatus>
+            <zoneName>us-west-2b</zoneName>
+            <zoneId>usw2-az2</zoneId>
+            <zoneState>available</zoneState>
+            <regionName>us-west-2</regionName>
+            <messageSet/>
+            <NetworkBorderGroup>us-west-2-lax-1</NetworkBorderGroup>
+        </item>
+    </availabilityZoneInfo>
+</DescribeAvailabilityZonesResponse>
+`
+
+	azr, err := parseAvailabilityZonesResponse([]byte(data))
+	if err != nil {
+		t.Fatalf("unexpected error when parsing data: %s", err)
+	}
+	azrExpected := &AvailabilityZonesResponse{
+		AvailabilityZoneInfo: AvailabilityZoneInfo{
+			Items: []AvailabilityZone{
+				{
+					ZoneName: "us-west-2a",
+					ZoneID:   "usw2-az1",
+				},
+				{
+					ZoneName: "us-west-2b",
+					ZoneID:   "usw2-az2",
+				},
+			},
+		},
+	}
+	if !reflect.DeepEqual(azr, azrExpected) {
+		t.Fatalf("unexpected DescribeAvailabilityZonesResponse parsed;\ngot\n%+v\nwant\n%+v", azr, azrExpected)
+	}
+}
+
 func TestParseInstancesResponse(t *testing.T) {
 	data := `<?xml version="1.0" encoding="UTF-8"?>
 <DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
@@ -188,31 +238,34 @@ func TestParseInstancesResponse(t *testing.T) {
 	ownerID := rs.OwnerID
 	port := 423
 	inst := rs.InstanceSet.Items[0]
-	labelss := inst.appendTargetLabels(nil, ownerID, port)
+	labelss := inst.appendTargetLabels(nil, ownerID, port, map[string]string{
+		"eu-west-2c": "foobar-zone",
+	})
 	var sortedLabelss [][]prompbmarshal.Label
 	for _, labels := range labelss {
 		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
 	}
 	expectedLabels := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
-			"__address__":                   "172.31.11.152:423",
-			"__meta_ec2_architecture":       "x86_64",
-			"__meta_ec2_availability_zone":  "eu-west-2c",
-			"__meta_ec2_ami":                "ami-0eb89db7593b5d434",
-			"__meta_ec2_instance_id":        "i-0e730b692d9c15460",
-			"__meta_ec2_instance_lifecycle": "spot",
-			"__meta_ec2_instance_state":     "running",
-			"__meta_ec2_instance_type":      "t2.micro",
-			"__meta_ec2_owner_id":           "793614593844",
-			"__meta_ec2_platform":           "windows",
-			"__meta_ec2_primary_subnet_id":  "subnet-57044c3e",
-			"__meta_ec2_private_dns_name":   "ip-172-31-11-152.eu-west-2.compute.internal",
-			"__meta_ec2_private_ip":         "172.31.11.152",
-			"__meta_ec2_public_dns_name":    "ec2-3-8-232-141.eu-west-2.compute.amazonaws.com",
-			"__meta_ec2_public_ip":          "3.8.232.141",
-			"__meta_ec2_subnet_id":          ",subnet-57044c3e,",
-			"__meta_ec2_tag_foo":            "bar",
-			"__meta_ec2_vpc_id":             "vpc-f1eaad99",
+			"__address__":                     "172.31.11.152:423",
+			"__meta_ec2_architecture":         "x86_64",
+			"__meta_ec2_availability_zone":    "eu-west-2c",
+			"__meta_ec2_availability_zone_id": "foobar-zone",
+			"__meta_ec2_ami":                  "ami-0eb89db7593b5d434",
+			"__meta_ec2_instance_id":          "i-0e730b692d9c15460",
+			"__meta_ec2_instance_lifecycle":   "spot",
+			"__meta_ec2_instance_state":       "running",
+			"__meta_ec2_instance_type":        "t2.micro",
+			"__meta_ec2_owner_id":             "793614593844",
+			"__meta_ec2_platform":             "windows",
+			"__meta_ec2_primary_subnet_id":    "subnet-57044c3e",
+			"__meta_ec2_private_dns_name":     "ip-172-31-11-152.eu-west-2.compute.internal",
+			"__meta_ec2_private_ip":           "172.31.11.152",
+			"__meta_ec2_public_dns_name":      "ec2-3-8-232-141.eu-west-2.compute.amazonaws.com",
+			"__meta_ec2_public_ip":            "3.8.232.141",
+			"__meta_ec2_subnet_id":            ",subnet-57044c3e,",
+			"__meta_ec2_tag_foo":              "bar",
+			"__meta_ec2_vpc_id":               "vpc-f1eaad99",
 		}),
 	}
 	if !reflect.DeepEqual(sortedLabelss, expectedLabels) {
diff --git a/lib/promscrape/discovery/gce/instance_test.go b/lib/promscrape/discovery/gce/instance_test.go
index a5426d81e..0364ae6ef 100644
--- a/lib/promscrape/discovery/gce/instance_test.go
+++ b/lib/promscrape/discovery/gce/instance_test.go
@@ -158,6 +158,7 @@ func TestParseInstanceListSuccess(t *testing.T) {
 			"__meta_gce_instance_id":                        "7897352091592122",
 			"__meta_gce_instance_name":                      "play-1m-1-vmagent",
 			"__meta_gce_instance_status":                    "RUNNING",
+			"__meta_gce_interface_ipv4_nic0":                "10.11.2.7",
 			"__meta_gce_label_cluster_num":                  "1",
 			"__meta_gce_label_cluster_retention":            "1m",
 			"__meta_gce_label_env":                          "play",
diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go
index 7d2eeced6..4083a4495 100644
--- a/lib/promscrape/scrapework_test.go
+++ b/lib/promscrape/scrapework_test.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -58,7 +59,7 @@ func TestScrapeWorkScrapeInternalFailure(t *testing.T) {
 
 	pushDataCalls := 0
 	var pushDataErr error
-	sw.PushData = func(wr *prompbmarshal.WriteRequest) {
+	sw.PushData = func(p *httpserver.Path, wr *prompbmarshal.WriteRequest) {
 		if err := expectEqualTimeseries(wr.Timeseries, timeseriesExpected); err != nil {
 			pushDataErr = fmt.Errorf("unexpected data pushed: %w\ngot\n%#v\nwant\n%#v", err, wr.Timeseries, timeseriesExpected)
 		}
@@ -98,7 +99,7 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) {
 
 		pushDataCalls := 0
 		var pushDataErr error
-		sw.PushData = func(wr *prompbmarshal.WriteRequest) {
+		sw.PushData = func(p *httpserver.Path, wr *prompbmarshal.WriteRequest) {
 			pushDataCalls++
 			if len(wr.Timeseries) > len(timeseriesExpected) {
 				pushDataErr = fmt.Errorf("too many time series obtained; got %d; want %d\ngot\n%+v\nwant\n%+v",
diff --git a/lib/promscrape/scrapework_timing_test.go b/lib/promscrape/scrapework_timing_test.go
index 0720d698e..0aa9ec363 100644
--- a/lib/promscrape/scrapework_timing_test.go
+++ b/lib/promscrape/scrapework_timing_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -39,7 +40,7 @@ vm_tcplistener_write_calls_total{name="https", addr=":443"} 132356
 		var sw scrapeWork
 		sw.Config = &ScrapeWork{}
 		sw.ReadData = readDataFunc
-		sw.PushData = func(wr *prompbmarshal.WriteRequest) {}
+		sw.PushData = func(p *httpserver.Path, wr *prompbmarshal.WriteRequest) {}
 		timestamp := int64(0)
 		for pb.Next() {
 			if err := sw.scrapeInternal(timestamp, timestamp); err != nil {