Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1283 (2021-05-11 10:59:16)

adds cgroupsv2 support


Patch:
diff --git a/README.md b/README.md
index 343228845..d71d5c9fb 100644
--- a/README.md
+++ b/README.md
@@ -549,9 +549,7 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
 * [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
 * [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
 * [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
-* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD`
-  query args for this handler, where `N` is the number of top entries to return in the response and `YYYY-MM-DD` is the date for collecting the stats.
-  By default top 10 entries are returned and the stats is collected for the current day.
+* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). See [these docs](#tsdb-stats) for details.
 * [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets) - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for more details.
 
 These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
@@ -1324,6 +1322,16 @@ VictoriaMetrics also exposes currently running queries with their execution time
 
 See the example of alerting rules for VM components [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
 
+
+## TSDB stats
+
+VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way similar to Prometheus - see [these Prometheus docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts the following optional query args at `/api/v1/status/tsdb` page:
+  * `topN=N` where `N` is the number of top entries to return in the response. By default top 10 entries are returned.
+  * `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
+  * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account.
+  * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details.
+
+
 ## Troubleshooting
 
 * It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
@@ -1380,10 +1388,7 @@ See the example of alerting rules for VM components [here](https://github.com/Vi
   It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes
   each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals.
 
-* Metrics and labels leading to high cardinality or high churn rate can be determined at `/api/v1/status/tsdb` page.
-  See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
-  VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
-  while `topN` equals to 10.
+* Metrics and labels leading to high cardinality or high churn rate can be determined at `/api/v1/status/tsdb` page. See [these docs](#tsdb-stats) for details.
 
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index c28a8b946..07c7f7a59 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -695,6 +695,29 @@ func GetTSDBStatusForDate(deadline searchutils.Deadline, date uint64, topN int)
 	return status, nil
 }
 
+// GetTSDBStatusWithFilters returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
+//
+// It accepts aribtrary filters on time series in sq.
+func GetTSDBStatusWithFilters(deadline searchutils.Deadline, sq *storage.SearchQuery, topN int) (*storage.TSDBStatus, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	tfss, err := setupTfss(tr, sq.TagFilterss, deadline)
+	if err != nil {
+		return nil, err
+	}
+	date := uint64(tr.MinTimestamp) / (3600 * 24 * 1000)
+	status, err := vmstorage.GetTSDBStatusWithFiltersForDate(tfss, date, topN, deadline.Deadline())
+	if err != nil {
+		return nil, fmt.Errorf("error during tsdb status with filters request: %w", err)
+	}
+	return status, nil
+}
+
 // GetSeriesCount returns the number of unique series.
 func GetSeriesCount(deadline searchutils.Deadline) (uint64, error) {
 	if deadline.Exceeded() {
diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index 137b625f1..0328a9648 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -633,11 +633,19 @@ const secsPerDay = 3600 * 24
 // TSDBStatusHandler processes /api/v1/status/tsdb request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
+//
+// It can accept `match[]` filters in order to narrow down the search.
 func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForStatusRequest(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
+	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	if err != nil {
+		return err
+	}
+	matches := getMatchesFromRequest(r)
+
 	date := fasttime.UnixDate()
 	dateStr := r.FormValue("date")
 	if len(dateStr) > 0 {
@@ -662,9 +670,17 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 		}
 		topN = n
 	}
-	status, err := netstorage.GetTSDBStatusForDate(deadline, date, topN)
-	if err != nil {
-		return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
+	var status *storage.TSDBStatus
+	if len(matches) == 0 && len(etf) == 0 {
+		status, err = netstorage.GetTSDBStatusForDate(deadline, date, topN)
+		if err != nil {
+			return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
+		}
+	} else {
+		status, err = tsdbStatusWithMatches(matches, etf, date, topN, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain tsdb status with matches for date=%d, topN=%d: %w", date, topN, err)
+		}
 	}
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	bw := bufferedwriter.Get(w)
@@ -677,6 +693,25 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	return nil
 }
 
+func tsdbStatusWithMatches(matches []string, etf []storage.TagFilter, date uint64, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return nil, err
+	}
+	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	if len(tagFilterss) == 0 {
+		logger.Panicf("BUG: tagFilterss must be non-empty")
+	}
+	start := int64(date*secsPerDay) * 1000
+	end := int64(date*secsPerDay+secsPerDay) * 1000
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
+	status, err := netstorage.GetTSDBStatusWithFilters(deadline, sq, topN)
+	if err != nil {
+		return nil, err
+	}
+	return status, nil
+}
+
 var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/tsdb"}`)
 
 // LabelsHandler processes /api/v1/labels request.
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index 731fc0002..7b052a206 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -213,6 +213,14 @@ func GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*storage.TSDB
 	return status, err
 }
 
+// GetTSDBStatusWithFiltersForDate returns TSDB status for given filters on the given date.
+func GetTSDBStatusWithFiltersForDate(tfss []*storage.TagFilters, date uint64, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	WG.Add(1)
+	status, err := Storage.GetTSDBStatusWithFiltersForDate(tfss, date, topN, deadline)
+	WG.Done()
+	return status, err
+}
+
 // GetSeriesCount returns the number of time series in the storage.
 func GetSeriesCount(deadline uint64) (uint64, error) {
 	WG.Add(1)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 806e77dcb..f21ccd808 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -7,6 +7,9 @@ sort: 15
 * FEATURE: vminsert: add support for data ingestion via other `vminsert` nodes. This allows building multi-level data ingestion paths in VictoriaMetrics cluster by writing data from one level of `vminsert` nodes to another level of `vminsert` nodes. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/541#issuecomment-835487858) for details.
 * FEATURE: vmalert: add flag to control behaviour on startup for state restore errors. Such errors were returned and logged before as well. But now user can specify whether to just log these errors (`-remoteRead.ignoreRestoreErrors=true`) or to stop the process (`-remoteRead.ignoreRestoreErrors=false`). The latter is important when VM isn't ready yet to serve queries from vmalert and it needs to wait. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1252).
 * FEATURE: vmalert: add ability to pass `round_digits` query arg to datasource via `-datasource.roundDigits` command-line flag. This can be used for limiting the number of decimal digits after the point in recording rule results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/525).
+* FEATURE: return `X-Server-Hostname` header in http responses of all the VictoriaMetrics components. This should simplify tracing the origin server behind a load balancer or behind auth proxy during troubleshooting.
+* FEATURE: vmselect: allow to use 2x more memory for query processing at `vmselect` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html). This should allow processing heavy queries without the need to increase RAM size at `vmselect` nodes.
+* FEATURE: add ability to filter `/api/v1/status/tsdb` output with arbitrary [time series selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) passed via `match[]` query args. See [these docs](https://docs.victoriametrics.com/#tsdb-stats) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1168) for details.
 
 * BUGFIX: vmagent: fix possible race when refreshing `role: endpoints` and `role: endpointslices` scrape targets in `kubernetes_sd_config`. Prevoiusly `pod` objects could be updated after the related `endpoints` object update. This could lead to missing scrape targets. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240).
 * BUGFIX: properly remove stale parts outside the configured retention if `-retentionPeriod` is smaller than one month. Previously stale parts could remain active for up to a month after they go outside the retention.
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index ec971cad2..128cc06a1 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -201,9 +201,7 @@ It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.co
     - `api/v1/export/native` - exports raw data in native binary format. It may be imported into another VictoriaMetrics via `api/v1/import/native` (see above).
     - `api/v1/export/csv` - exports data in CSV. It may be imported into another VictoriaMetrics via `api/v1/import/csv` (see above).
     - `api/v1/series/count` - returns the total number of series.
-    - `api/v1/status/tsdb` - for time series stats. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
-      VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD` query args for this handler, where `N` is the number of top entries to return in the response
-      and `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
+    - `api/v1/status/tsdb` - for time series stats. See [these docs](https://docs.victoriametrics.com/#tsdb-stats) for details.
     - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries,
       which is returned in the response.
     - `api/v1/status/top_queries` - for listing the most frequently executed queries and queries taking the most duration.
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 64657442e..6cc56b94c 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -553,9 +553,7 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
 * [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
 * [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
 * [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
-* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD`
-  query args for this handler, where `N` is the number of top entries to return in the response and `YYYY-MM-DD` is the date for collecting the stats.
-  By default top 10 entries are returned and the stats is collected for the current day.
+* [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). See [these docs](#tsdb-stats) for details.
 * [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets) - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for more details.
 
 These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
@@ -1328,6 +1326,16 @@ VictoriaMetrics also exposes currently running queries with their execution time
 
 See the example of alerting rules for VM components [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
 
+
+## TSDB stats
+
+VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way similar to Prometheus - see [these Prometheus docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts the following optional query args at `/api/v1/status/tsdb` page:
+  * `topN=N` where `N` is the number of top entries to return in the response. By default top 10 entries are returned.
+  * `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
+  * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account.
+  * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details.
+
+
 ## Troubleshooting
 
 * It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
@@ -1384,10 +1392,7 @@ See the example of alerting rules for VM components [here](https://github.com/Vi
   It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes
   each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals.
 
-* Metrics and labels leading to high cardinality or high churn rate can be determined at `/api/v1/status/tsdb` page.
-  See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
-  VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
-  while `topN` equals to 10.
+* Metrics and labels leading to high cardinality or high churn rate can be determined at `/api/v1/status/tsdb` page. See [these docs](#tsdb-stats) for details.
 
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
diff --git a/lib/cgroup/cpu.go b/lib/cgroup/cpu.go
index 894f4dbdb..8c3972d50 100644
--- a/lib/cgroup/cpu.go
+++ b/lib/cgroup/cpu.go
@@ -1,6 +1,7 @@
 package cgroup
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"runtime"
@@ -41,20 +42,28 @@ func updateGOMAXPROCSToCPUQuota() {
 }
 
 func getCPUQuota() float64 {
-	quotaUS, err := getCPUStat("cpu.cfs_quota_us")
+	cpuQuota, err := getCPUStatGeneric()
 	if err != nil {
 		return 0
 	}
-	if quotaUS <= 0 {
+
+	if cpuQuota <= 0 {
 		// The quota isn't set. This may be the case in multilevel containers.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728
 		return getOnlineCPUCount()
 	}
-	periodUS, err := getCPUStat("cpu.cfs_period_us")
-	if err != nil {
-		return 0
+	return cpuQuota
+}
+
+func getCPUStatGeneric() (float64, error) {
+	quotaUS, err := getCPUStat("cpu.cfs_quota_us")
+	if err == nil {
+		periodUS, err := getCPUStat("cpu.cfs_period_us")
+		if err == nil {
+			return float64(quotaUS) / float64(periodUS), nil
+		}
 	}
-	return float64(quotaUS) / float64(periodUS)
+	return getCPUStatV2("/sys/fs/cgroup", "/proc/self/cgroup")
 }
 
 func getCPUStat(statName string) (int64, error) {
@@ -74,6 +83,35 @@ func getOnlineCPUCount() float64 {
 	return n
 }
 
+func getCPUStatV2(sysPrefix, cgroupPath string) (float64, error) {
+	data, err := getFileContents("cpu.max", sysPrefix, cgroupPath, "")
+	if err != nil {
+		return 0, err
+	}
+	return parseCPUMax(data)
+}
+
+// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#cpu
+func parseCPUMax(data string) (float64, error) {
+	data = strings.TrimRight(data, "\r\n")
+	bounds := strings.Split(data, " ")
+	if len(bounds) != 2 {
+		return 0, fmt.Errorf("unexpected count: %d, want quota and period, got: %s", len(bounds), data)
+	}
+	if bounds[0] == "max" {
+		return -1, nil
+	}
+	quota, err := strconv.ParseUint(bounds[0], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	period, err := strconv.ParseUint(bounds[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return float64(quota) / float64(period), nil
+}
+
 func countCPUs(data string) int {
 	data = strings.TrimSpace(data)
 	n := 0
diff --git a/lib/cgroup/mem.go b/lib/cgroup/mem.go
index 92ac92b70..e91f8145b 100644
--- a/lib/cgroup/mem.go
+++ b/lib/cgroup/mem.go
@@ -13,12 +13,22 @@ func GetMemoryLimit() int64 {
 	// This should properly determine the limit inside lxc container.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/84
 	n, err := getMemStat("memory.limit_in_bytes")
+	if err == nil {
+		return n
+	}
+	// https: //www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files
+	n, err = getMemStatV2()
 	if err != nil {
 		return 0
 	}
+
 	return n
 }
 
+func getMemStatV2() (int64, error) {
+	return getStatGeneric("memory.max", "/sys/fs/cgroup", "/proc/self/cgroup", "")
+}
+
 func getMemStat(statName string) (int64, error) {
 	return getStatGeneric(statName, "/sys/fs/cgroup/memory", "/proc/self/cgroup", "memory")
 }
diff --git a/lib/cgroup/util.go b/lib/cgroup/util.go
index 3dcd64e4c..66960adb7 100644
--- a/lib/cgroup/util.go
+++ b/lib/cgroup/util.go
@@ -13,6 +13,7 @@ func getStatGeneric(statName, sysfsPrefix, cgroupPath, cgroupGrepLine string) (i
 	if err != nil {
 		return 0, err
 	}
+	data = strings.TrimRight(data, "\r\n")
 	n, err := strconv.ParseInt(data, 10, 64)
 	if err != nil {
 		return 0, err
diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go
index e2892ca99..d3f4260fc 100644
--- a/lib/httpserver/httpserver.go
+++ b/lib/httpserver/httpserver.go
@@ -8,6 +8,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"net/http/pprof"
@@ -194,6 +195,17 @@ func gzipHandler(s *server, rh RequestHandler) http.HandlerFunc {
 var metricsHandlerDuration = metrics.NewHistogram(`vm_http_request_duration_seconds{path="/metrics"}`)
 var connTimeoutClosedConns = metrics.NewCounter(`vm_http_conn_timeout_closed_conns_total`)
 
+var hostname = func() string {
+	h, err := os.Hostname()
+	if err != nil {
+		// Cannot use logger.Errorf, since it isn't initialized yet.
+		// So use log.Printf instead.
+		log.Printf("ERROR: cannot determine hostname: %s", err)
+		return "unknown"
+	}
+	return h
+}()
+
 func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh RequestHandler) {
 	// All the VictoriaMetrics code assumes that panic stops the process.
 	// Unfortunately, the standard net/http.Server recovers from panics in request handlers,
@@ -209,6 +221,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		}
 	}()
 
+	w.Header().Add("X-Server-Hostname", hostname)
 	requestsTotal.Inc()
 	if whetherToCloseConn(r) {
 		connTimeoutClosedConns.Inc()
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index 854a510fe..1dbbbc1cc 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -75,11 +75,6 @@ func newAPIWatcher(apiServer string, ac *promauth.Config, sdc *SDConfig, swcFunc
 }
 
 func (aw *apiWatcher) mustStart() {
-	if aw.role == "endpoints" || aw.role == "endpointslices" {
-		// endpoints and endpointslices watchers query pod and service objects. So start watchers for these roles as well.
-		aw.gw.startWatchersForRole("pod", nil)
-		aw.gw.startWatchersForRole("service", nil)
-	}
 	aw.gw.startWatchersForRole(aw.role, aw)
 }
 
@@ -255,37 +250,12 @@ func (gw *groupWatcher) getObjectByRoleLocked(role, namespace, name string) obje
 	return nil
 }
 
-func (gw *groupWatcher) refreshEndpointsLabelsLocked(namespace, key string) {
-	// Refresh endpoints and endpointslices labels for the corresponding service as Prometheus does.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-	gw.refreshObjectLabelsLocked("endpoints", namespace, key)
-	gw.refreshObjectLabelsLocked("endpointslices", namespace, key)
-}
-
-func (gw *groupWatcher) refreshObjectLabelsLocked(role, namespace, key string) {
-	for _, uw := range gw.m {
-		if len(uw.aws) == 0 {
-			// No apiWatchers to notify
-			continue
-		}
-		if uw.role != role {
-			// Role mismatch
-			continue
-		}
-		if uw.namespace != "" && uw.namespace != namespace {
-			// Namespace mismatch
-			continue
-		}
-		if o := uw.objectsByKey[key]; o != nil {
-			labels := o.getTargetLabels(gw)
-			for aw := range uw.aws {
-				aw.setScrapeWorks(namespace, key, labels)
-			}
-		}
-	}
-}
-
 func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
+	if role == "endpoints" || role == "endpointslices" {
+		// endpoints and endpointslices watchers query pod and service objects. So start watchers for these roles as well.
+		gw.startWatchersForRole("pod", nil)
+		gw.startWatchersForRole("service", nil)
+	}
 	paths, namespaces := getAPIPathsWithNamespaces(role, gw.namespaces, gw.selectors)
 	for i, path := range paths {
 		apiURL := gw.apiServer + path
@@ -303,6 +273,18 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 		if needStart {
 			uw.reloadObjects()
 			go uw.watchForUpdates()
+			if role == "endpoints" || role == "endpointslices" {
+				// Refresh endpoints and enpointslices targets in background, since they depend on other object types such as pod and service.
+				// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240 .
+				go func() {
+					for {
+						time.Sleep(5 * time.Second)
+						gw.mu.Lock()
+						uw.reloadScrapeWorksForAPIWatchersLocked(uw.aws)
+						gw.mu.Unlock()
+					}
+				}()
+			}
 		}
 	}
 }
@@ -488,13 +470,6 @@ func (uw *urlWatcher) reloadObjects() string {
 	uw.objectsCount.Add(added - removed)
 	uw.reloadScrapeWorksForAPIWatchersLocked(uw.aws)
 	uw.resourceVersion = metadata.ResourceVersion
-	if uw.role == "service" {
-		// Refresh endpoints labels for the corresponding services as Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-		for key := range objectsByKey {
-			uw.gw.refreshEndpointsLabelsLocked(uw.namespace, key)
-		}
-	}
 	uw.gw.mu.Unlock()
 
 	logger.Infof("reloaded %d objects from %q", len(objectsByKey), requestURL)
@@ -616,11 +591,6 @@ func (uw *urlWatcher) readObjectUpdateStream(r io.Reader) error {
 					aw.setScrapeWorks(uw.namespace, key, labels)
 				}
 			}
-			if uw.role == "service" {
-				// Refresh endpoints labels for the corresponding service as Prometheus does.
-				// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-				uw.gw.refreshEndpointsLabelsLocked(uw.namespace, key)
-			}
 			uw.gw.mu.Unlock()
 		case "DELETED":
 			o, err := uw.parseObject(we.Object)
@@ -637,11 +607,6 @@ func (uw *urlWatcher) readObjectUpdateStream(r io.Reader) error {
 			for aw := range uw.aws {
 				aw.removeScrapeWorks(uw.namespace, key)
 			}
-			if uw.role == "service" {
-				// Refresh endpoints labels for the corresponding service as Prometheus does.
-				// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-				uw.gw.refreshEndpointsLabelsLocked(uw.namespace, key)
-			}
 			uw.gw.mu.Unlock()
 		case "BOOKMARK":
 			// See https://kubernetes.io/docs/reference/using-api/api-concepts/#watch-bookmarks
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 488f763e3..5b013e86d 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -1309,23 +1309,20 @@ func (is *indexSearch) getSeriesCount() (uint64, error) {
 	return metricIDsLen, nil
 }
 
-// GetTSDBStatusForDate returns topN entries for tsdb status for the given date.
-func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*TSDBStatus, error) {
+// GetTSDBStatusWithFiltersForDate returns topN entries for tsdb status for the given tfss and the given date.
+func (db *indexDB) GetTSDBStatusWithFiltersForDate(tfss []*TagFilters, date uint64, topN int, deadline uint64) (*TSDBStatus, error) {
 	is := db.getIndexSearch(deadline)
-	status, err := is.getTSDBStatusForDate(date, topN)
+	status, err := is.getTSDBStatusWithFiltersForDate(tfss, date, topN)
 	db.putIndexSearch(is)
 	if err != nil {
 		return nil, err
 	}
 	if status.hasEntries() {
-		// The entries were found in the db. There is no need in searching them in extDB.
 		return status, nil
 	}
-
-	// The entries weren't found in the db. Try searching them in extDB.
 	ok := db.doExtDB(func(extDB *indexDB) {
 		is := extDB.getIndexSearch(deadline)
-		status, err = is.getTSDBStatusForDate(date, topN)
+		status, err = is.getTSDBStatusWithFiltersForDate(tfss, date, topN)
 		extDB.putIndexSearch(is)
 	})
 	if ok && err != nil {
@@ -1334,7 +1331,25 @@ func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int, deadline uint64)
 	return status, nil
 }
 
-func (is *indexSearch) getTSDBStatusForDate(date uint64, topN int) (*TSDBStatus, error) {
+// getTSDBStatusWithFiltersForDate returns topN entries for tsdb status for the given tfss and the given date.
+func (is *indexSearch) getTSDBStatusWithFiltersForDate(tfss []*TagFilters, date uint64, topN int) (*TSDBStatus, error) {
+	var filter *uint64set.Set
+	if len(tfss) > 0 {
+		tr := TimeRange{
+			MinTimestamp: int64(date) * msecPerDay,
+			MaxTimestamp: int64(date+1) * msecPerDay,
+		}
+		metricIDs, err := is.searchMetricIDsInternal(tfss, tr, 2e9)
+		if err != nil {
+			return nil, err
+		}
+		if metricIDs.Len() == 0 {
+			// Nothing found.
+			return &TSDBStatus{}, nil
+		}
+		filter = metricIDs
+	}
+
 	ts := &is.ts
 	kb := &is.kb
 	mp := &is.mp
@@ -1361,6 +1376,22 @@ func (is *indexSearch) getTSDBStatusForDate(date uint64, topN int) (*TSDBStatus,
 		if !bytes.HasPrefix(item, prefix) {
 			break
 		}
+		matchingSeriesCount := 0
+		if filter != nil {
+			if err := mp.Init(item, nsPrefixDateTagToMetricIDs); err != nil {
+				return nil, err
+			}
+			mp.ParseMetricIDs()
+			for _, metricID := range mp.MetricIDs {
+				if filter.Has(metricID) {
+					matchingSeriesCount++
+				}
+			}
+			if matchingSeriesCount == 0 {
+				// Skip rows without matching metricIDs.
+				continue
+			}
+		}
 		tail := item[len(prefix):]
 		var err error
 		tail, tmp, err = unmarshalTagValue(tmp[:0], tail)
@@ -1393,13 +1424,16 @@ func (is *indexSearch) getTSDBStatusForDate(date uint64, topN int) (*TSDBStatus,
 			labelValueCountByLabelName++
 			labelNameValue = append(labelNameValue[:0], tmp...)
 		}
-		if err := mp.InitOnlyTail(item, tail); err != nil {
-			return nil, err
+		if filter == nil {
+			if err := mp.InitOnlyTail(item, tail); err != nil {
+				return nil, err
+			}
+			matchingSeriesCount = mp.MetricIDsLen()
 		}
 		// Take into account deleted timeseries too.
 		// It is OK if series can be counted multiple times in rare cases -
 		// the returned number is an estimation.
-		seriesCountByLabelValuePair += uint64(mp.MetricIDsLen())
+		seriesCountByLabelValuePair += uint64(matchingSeriesCount)
 	}
 	if err := ts.Error(); err != nil {
 		return nil, fmt.Errorf("error when counting time series by metric names: %w", err)
@@ -1417,6 +1451,31 @@ func (is *indexSearch) getTSDBStatusForDate(date uint64, topN int) (*TSDBStatus,
 	return status, nil
 }
 
+// GetTSDBStatusForDate returns topN entries for tsdb status for the given date.
+func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*TSDBStatus, error) {
+	is := db.getIndexSearch(deadline)
+	status, err := is.getTSDBStatusWithFiltersForDate(nil, date, topN)
+	db.putIndexSearch(is)
+	if err != nil {
+		return nil, err
+	}
+	if status.hasEntries() {
+		// The entries were found in the db. There is no need in searching them in extDB.
+		return status, nil
+	}
+
+	// The entries weren't found in the db. Try searching them in extDB.
+	ok := db.doExtDB(func(extDB *indexDB) {
+		is := extDB.getIndexSearch(deadline)
+		status, err = is.getTSDBStatusWithFiltersForDate(nil, date, topN)
+		extDB.putIndexSearch(is)
+	})
+	if ok && err != nil {
+		return nil, fmt.Errorf("error when obtaining TSDB status from extDB: %w", err)
+	}
+	return status, nil
+}
+
 // TSDBStatus contains TSDB status data for /api/v1/status/tsdb.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
@@ -2290,21 +2349,9 @@ func matchTagFilters(mn *MetricName, tfs []*tagFilter, kb *bytesutil.ByteBuffer)
 }
 
 func (is *indexSearch) searchMetricIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) ([]uint64, error) {
-	metricIDs := &uint64set.Set{}
-	for _, tfs := range tfss {
-		if len(tfs.tfs) == 0 {
-			// An empty filters must be equivalent to `{__name__!=""}`
-			tfs = NewTagFilters()
-			if err := tfs.Add(nil, nil, true, false); err != nil {
-				logger.Panicf(`BUG: cannot add {__name__!=""} filter: %s`, err)
-			}
-		}
-		if err := is.updateMetricIDsForTagFilters(metricIDs, tfs, tr, maxMetrics+1); err != nil {
-			return nil, err
-		}
-		if metricIDs.Len() > maxMetrics {
-			return nil, fmt.Errorf("the number of matching unique timeseries exceeds %d; either narrow down the search or increase -search.maxUniqueTimeseries", maxMetrics)
-		}
+	metricIDs, err := is.searchMetricIDsInternal(tfss, tr, maxMetrics)
+	if err != nil {
+		return nil, err
 	}
 	if metricIDs.Len() == 0 {
 		// Nothing found
@@ -2328,6 +2375,26 @@ func (is *indexSearch) searchMetricIDs(tfss []*TagFilters, tr TimeRange, maxMetr
 	return sortedMetricIDs, nil
 }
 
+func (is *indexSearch) searchMetricIDsInternal(tfss []*TagFilters, tr TimeRange, maxMetrics int) (*uint64set.Set, error) {
+	metricIDs := &uint64set.Set{}
+	for _, tfs := range tfss {
+		if len(tfs.tfs) == 0 {
+			// An empty filters must be equivalent to `{__name__!=""}`
+			tfs = NewTagFilters()
+			if err := tfs.Add(nil, nil, true, false); err != nil {
+				logger.Panicf(`BUG: cannot add {__name__!=""} filter: %s`, err)
+			}
+		}
+		if err := is.updateMetricIDsForTagFilters(metricIDs, tfs, tr, maxMetrics+1); err != nil {
+			return nil, err
+		}
+		if metricIDs.Len() > maxMetrics {
+			return nil, fmt.Errorf("the number of matching unique timeseries exceeds %d; either narrow down the search or increase -search.maxUniqueTimeseries", maxMetrics)
+		}
+	}
+	return metricIDs, nil
+}
+
 func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs *uint64set.Set, tfs *TagFilters, tr TimeRange, maxMetrics int) error {
 	err := is.tryUpdatingMetricIDsForDateRange(metricIDs, tfs, tr, maxMetrics)
 	if err == nil {
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index 81bd3b18e..772204ca4 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -1248,6 +1248,11 @@ func (s *Storage) GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (
 	return s.idb().GetTSDBStatusForDate(date, topN, deadline)
 }
 
+// GetTSDBStatusWithFiltersForDate returns TSDB status data for /api/v1/status/tsdb with match[] filters.
+func (s *Storage) GetTSDBStatusWithFiltersForDate(tfss []*TagFilters, date uint64, topN int, deadline uint64) (*TSDBStatus, error) {
+	return s.idb().GetTSDBStatusWithFiltersForDate(tfss, date, topN, deadline)
+}
+
 // MetricRow is a metric to insert into storage.
 type MetricRow struct {
 	// MetricNameRaw contains raw metric name, which must be decoded


Test patch:
diff --git a/lib/cgroup/cpu_test.go b/lib/cgroup/cpu_test.go
index 37aaceb6c..20084b305 100644
--- a/lib/cgroup/cpu_test.go
+++ b/lib/cgroup/cpu_test.go
@@ -22,3 +22,17 @@ func TestCountCPUs(t *testing.T) {
 	f("0-3", 4)
 	f("0-6", 7)
 }
+
+func TestGetCPUStatV2(t *testing.T) {
+	f := func(sysPrefix, cgroupPath string, expectedCPU float64) {
+		t.Helper()
+		got, err := getCPUStatV2(sysPrefix, cgroupPath)
+		if err != nil {
+			t.Fatalf("unexpected error: %s, sysPrefix: %s, cgroupPath: %s", err, sysPrefix, cgroupPath)
+		}
+		if got != expectedCPU {
+			t.Fatalf("unexpected result from getCPUStatV2(%s, %s), got %f, want %f", sysPrefix, cgroupPath, got, expectedCPU)
+		}
+	}
+	f("testdata/cgroup", "testdata/self/cgroupv2", 2)
+}
diff --git a/lib/cgroup/testdata/cgroup/cpu.max b/lib/cgroup/testdata/cgroup/cpu.max
new file mode 100644
index 000000000..96856569d
--- /dev/null
+++ b/lib/cgroup/testdata/cgroup/cpu.max
@@ -0,0 +1,1 @@
+200000 100000
diff --git a/lib/cgroup/testdata/cgroup/memory.max b/lib/cgroup/testdata/cgroup/memory.max
new file mode 100644
index 000000000..4f58557a7
--- /dev/null
+++ b/lib/cgroup/testdata/cgroup/memory.max
@@ -0,0 +1,1 @@
+523372036854771712
diff --git a/lib/cgroup/testdata/self/cgroupv2 b/lib/cgroup/testdata/self/cgroupv2
new file mode 100644
index 000000000..62bb6b637
--- /dev/null
+++ b/lib/cgroup/testdata/self/cgroupv2
@@ -0,0 +1,1 @@
+0::/
\ No newline at end of file
diff --git a/lib/cgroup/util_test.go b/lib/cgroup/util_test.go
index b6123e641..a4bc2d590 100644
--- a/lib/cgroup/util_test.go
+++ b/lib/cgroup/util_test.go
@@ -21,6 +21,7 @@ func TestGetStatGenericSuccess(t *testing.T) {
 	f("cpu.cfs_period_us", "testdata/cgroup", "testdata/self/cgroup", "cpu,", 500000)
 	f("memory.limit_in_bytes", "testdata/", "testdata/self/cgroup", "memory", 9223372036854771712)
 	f("memory.limit_in_bytes", "testdata/cgroup", "testdata/self/cgroup", "memory", 523372036854771712)
+	f("memory.max", "testdata/cgroup", "testdata/self/cgroupv2", "", 523372036854771712)
 }
 
 func TestGetStatGenericFailure(t *testing.T) {
@@ -37,4 +38,5 @@ func TestGetStatGenericFailure(t *testing.T) {
 	f("cpu.cfs_quota_us", "testdata/", "testdata/missing_folder", "cpu,")
 	f("cpu.cfs_period_us", "testdata/", "testdata/missing_folder", "cpu,")
 	f("memory.limit_in_bytes", "testdata/", "testdata/none_existing_folder", "memory")
+	f("memory.max", "testdata/", "testdata/none_existing_folder", "")
 }
diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go
index 98a4943d5..a9bbbe847 100644
--- a/lib/storage/index_db_test.go
+++ b/lib/storage/index_db_test.go
@@ -1691,6 +1691,28 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
 	if !reflect.DeepEqual(status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair) {
 		t.Fatalf("unexpected SeriesCountByLabelValuePair;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair)
 	}
+
+	// Check GetTSDBStatusWithFiltersForDate
+	tfs = NewTagFilters()
+	if err := tfs.Add([]byte("day"), []byte("0"), false, false); err != nil {
+		t.Fatalf("cannot add filter: %s", err)
+	}
+	status, err = db.GetTSDBStatusWithFiltersForDate([]*TagFilters{tfs}, baseDate, 5, noDeadline)
+	if err != nil {
+		t.Fatalf("error in GetTSDBStatusWithFiltersForDate: %s", err)
+	}
+	if !status.hasEntries() {
+		t.Fatalf("expecting non-empty TSDB status")
+	}
+	expectedSeriesCountByMetricName = []TopHeapEntry{
+		{
+			Name:  "testMetric",
+			Count: 1000,
+		},
+	}
+	if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
+		t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
+	}
 }
 
 func toTFPointers(tfs []tagFilter) []*tagFilter {