Repository: VictoriaMetrics/VictoriaMetrics
Pull request #140 (2019-08-03 20:19:54)

[deployment] add statefulset for vmselect

tenmozes: @zunkree  could you review?

Patch:
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index a199f399b..5f61882c3 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -45,9 +45,10 @@ func (r *Result) reset() {
 
 // Results holds results returned from ProcessSearchQuery.
 type Results struct {
-	at       *auth.Token
-	tr       storage.TimeRange
-	deadline Deadline
+	at        *auth.Token
+	tr        storage.TimeRange
+	fetchData bool
+	deadline  Deadline
 
 	tbf *tmpBlocksFile
 
@@ -100,10 +101,10 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.Timeout)
 					break
 				}
-				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.at, maxWorkersCount); err != nil {
+				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.fetchData, rss.at, maxWorkersCount); err != nil {
 					break
 				}
-				if len(rs.Timestamps) == 0 {
+				if len(rs.Timestamps) == 0 && rss.fetchData {
 					// Skip empty blocks.
 					continue
 				}
@@ -146,7 +147,7 @@ type packedTimeseries struct {
 }
 
 // Unpack unpacks pts to dst.
-func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, at *auth.Token, maxWorkersCount int) error {
+func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, fetchData bool, at *auth.Token, maxWorkersCount int) error {
 	dst.reset()
 
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
@@ -173,7 +174,7 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 			var err error
 			for addr := range workCh {
 				sb := getSortBlock()
-				if err = sb.unpackFrom(tbf, addr, tr, at); err != nil {
+				if err = sb.unpackFrom(tbf, addr, tr, fetchData, at); err != nil {
 					break
 				}
 
@@ -292,10 +293,12 @@ func (sb *sortBlock) reset() {
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange, at *auth.Token) error {
+func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange, fetchData bool, at *auth.Token) error {
 	tbf.MustReadBlockAt(&sb.b, addr)
-	if err := sb.b.UnmarshalData(); err != nil {
-		return fmt.Errorf("cannot unmarshal block: %s", err)
+	if fetchData {
+		if err := sb.b.UnmarshalData(); err != nil {
+			return fmt.Errorf("cannot unmarshal block: %s", err)
+		}
 	}
 	timestamps := sb.b.Timestamps()
 
@@ -692,7 +695,7 @@ func GetSeriesCount(at *auth.Token, deadline Deadline) (uint64, bool, error) {
 }
 
 // ProcessSearchQuery performs sq on storage nodes until the given deadline.
-func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, deadline Deadline) (*Results, bool, error) {
+func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, bool, error) {
 	requestData := sq.Marshal(nil)
 
 	// Send the query to all the storage nodes in parallel.
@@ -708,7 +711,7 @@ func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, deadline Deadli
 	for _, sn := range storageNodes {
 		go func(sn *storageNode) {
 			sn.searchRequests.Inc()
-			results, err := sn.processSearchQuery(requestData, tr, deadline)
+			results, err := sn.processSearchQuery(requestData, tr, fetchData, deadline)
 			if err != nil {
 				sn.searchRequestErrors.Inc()
 				err = fmt.Errorf("cannot perform search on vmstorage %s: %s", sn.connPool.Addr(), err)
@@ -769,6 +772,7 @@ func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, deadline Deadli
 	rss.packedTimeseries = make([]packedTimeseries, len(m))
 	rss.at = at
 	rss.tr = tr
+	rss.fetchData = fetchData
 	rss.deadline = deadline
 	rss.tbf = tbf
 	i := 0
@@ -931,20 +935,20 @@ func (sn *storageNode) getSeriesCount(accountID, projectID uint32, deadline Dead
 	return n, nil
 }
 
-func (sn *storageNode) processSearchQuery(requestData []byte, tr storage.TimeRange, deadline Deadline) ([]*storage.MetricBlock, error) {
+func (sn *storageNode) processSearchQuery(requestData []byte, tr storage.TimeRange, fetchData bool, deadline Deadline) ([]*storage.MetricBlock, error) {
 	var results []*storage.MetricBlock
 	f := func(bc *handshake.BufferedConn) error {
-		rs, err := sn.processSearchQueryOnConn(bc, requestData, tr)
+		rs, err := sn.processSearchQueryOnConn(bc, requestData, tr, fetchData)
 		if err != nil {
 			return err
 		}
 		results = rs
 		return nil
 	}
-	if err := sn.execOnConn("search_v2", f, deadline); err != nil {
+	if err := sn.execOnConn("search_v3", f, deadline); err != nil {
 		// Try again before giving up.
 		results = nil
-		if err = sn.execOnConn("search_v2", f, deadline); err != nil {
+		if err = sn.execOnConn("search_v3", f, deadline); err != nil {
 			return nil, err
 		}
 	}
@@ -1197,11 +1201,14 @@ const maxMetricBlockSize = 1024 * 1024
 // from vmstorage.
 const maxErrorMessageSize = 64 * 1024
 
-func (sn *storageNode) processSearchQueryOnConn(bc *handshake.BufferedConn, requestData []byte, tr storage.TimeRange) ([]*storage.MetricBlock, error) {
+func (sn *storageNode) processSearchQueryOnConn(bc *handshake.BufferedConn, requestData []byte, tr storage.TimeRange, fetchData bool) ([]*storage.MetricBlock, error) {
 	// Send the request to sn.
 	if err := writeBytes(bc, requestData); err != nil {
 		return nil, fmt.Errorf("cannot write requestData: %s", err)
 	}
+	if err := writeBool(bc, fetchData); err != nil {
+		return nil, fmt.Errorf("cannot write fetchData=%v: %s", fetchData, err)
+	}
 	if err := bc.Flush(); err != nil {
 		return nil, fmt.Errorf("cannot flush requestData to conn: %s", err)
 	}
@@ -1265,6 +1272,17 @@ func writeUint32(bc *handshake.BufferedConn, n uint32) error {
 	return nil
 }
 
+func writeBool(bc *handshake.BufferedConn, b bool) error {
+	var buf [1]byte
+	if b {
+		buf[0] = 1
+	}
+	if _, err := bc.Write(buf[:]); err != nil {
+		return err
+	}
+	return nil
+}
+
 func readBytes(buf []byte, bc *handshake.BufferedConn, maxDataSize int) ([]byte, error) {
 	buf = bytesutil.Resize(buf, 8)
 	if _, err := io.ReadFull(bc, buf); err != nil {
diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index 71638815f..de2e78d53 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -6,8 +6,10 @@ import (
 	"math"
 	"net/http"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
@@ -72,7 +74,7 @@ func FederateHandler(at *auth.Token, w http.ResponseWriter, r *http.Request) err
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, deadline)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, true, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
@@ -169,7 +171,7 @@ func exportHandler(at *auth.Token, w http.ResponseWriter, matches []string, star
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, deadline)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, true, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
@@ -284,9 +286,40 @@ var httpClient = &http.Client{
 func LabelValuesHandler(at *auth.Token, labelName string, w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
 	deadline := getDeadline(r)
-	labelValues, isPartial, err := netstorage.GetLabelValues(at, labelName, deadline)
-	if err != nil {
-		return fmt.Errorf(`cannot obtain label values for %q: %s`, labelName, err)
+
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %s", err)
+	}
+	var labelValues []string
+	var isPartial bool
+	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+		var err error
+		labelValues, isPartial, err = netstorage.GetLabelValues(at, labelName, deadline)
+		if err != nil {
+			return fmt.Errorf(`cannot obtain label values for %q: %s`, labelName, err)
+		}
+	} else {
+		// Extended functionality that allows filtering by label filters and time range
+		// i.e. /api/v1/label/foo/values?match[]=foobar{baz="abc"}&start=...&end=...
+		// is equivalent to `label_values(foobar{baz="abc"}, foo)` call on the selected
+		// time range in Grafana templating.
+		matches := r.Form["match[]"]
+		if len(matches) == 0 {
+			matches = []string{fmt.Sprintf("{%s!=''}", labelName)}
+		}
+		ct := currentTime()
+		end, err := getTime(r, "end", ct)
+		if err != nil {
+			return err
+		}
+		start, err := getTime(r, "start", end-defaultStep)
+		if err != nil {
+			return err
+		}
+		labelValues, isPartial, err = labelValuesWithMatches(at, labelName, matches, start, end, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %s", labelName, matches, start, end, err)
+		}
 	}
 	if isPartial && getDenyPartialResponse(r) {
 		return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
@@ -298,6 +331,52 @@ func LabelValuesHandler(at *auth.Token, labelName string, w http.ResponseWriter,
 	return nil
 }
 
+func labelValuesWithMatches(at *auth.Token, labelName string, matches []string, start, end int64, deadline netstorage.Deadline) ([]string, bool, error) {
+	if len(matches) == 0 {
+		logger.Panicf("BUG: matches must be non-empty")
+	}
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return nil, false, err
+	}
+	if start >= end {
+		start = end - defaultStep
+	}
+	sq := &storage.SearchQuery{
+		AccountID:    at.AccountID,
+		ProjectID:    at.ProjectID,
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, false, deadline)
+	if err != nil {
+		return nil, false, fmt.Errorf("cannot fetch data for %q: %s", sq, err)
+	}
+
+	m := make(map[string]struct{})
+	var mLock sync.Mutex
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		labelValue := rs.MetricName.GetTagValue(labelName)
+		if len(labelValue) == 0 {
+			return
+		}
+		mLock.Lock()
+		m[string(labelValue)] = struct{}{}
+		mLock.Unlock()
+	})
+	if err != nil {
+		return nil, false, fmt.Errorf("error when data fetching: %s", err)
+	}
+
+	labelValues := make([]string, 0, len(m))
+	for labelValue := range m {
+		labelValues = append(labelValues, labelValue)
+	}
+	sort.Strings(labelValues)
+	return labelValues, isPartial, nil
+}
+
 var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/label/{}/values"}`)
 
 // LabelsCountHandler processes /api/v1/labels/count request.
@@ -376,13 +455,16 @@ func SeriesHandler(at *auth.Token, w http.ResponseWriter, r *http.Request) error
 	if len(matches) == 0 {
 		return fmt.Errorf("missing `match[]` arg")
 	}
-	// Set start to minTimeMsecs by default as Prometheus does.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
-	start, err := getTime(r, "start", minTimeMsecs)
+	end, err := getTime(r, "end", ct)
 	if err != nil {
 		return err
 	}
-	end, err := getTime(r, "end", ct)
+	// Do not set start to minTimeMsecs by default as Prometheus does,
+	// since this leads to fetching and scanning all the data from the storage,
+	// which can take a lot of time for big storages.
+	// It is better setting start as end-defaultStep by default.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
+	start, err := getTime(r, "start", end-defaultStep)
 	if err != nil {
 		return err
 	}
@@ -402,7 +484,7 @@ func SeriesHandler(at *auth.Token, w http.ResponseWriter, r *http.Request) error
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, deadline)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, false, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go
index ead47a162..50062e4b5 100644
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -578,8 +578,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 		MaxTimestamp: ec.End + ec.Step,
 		TagFilterss:  [][]storage.TagFilter{me.TagFilters},
 	}
-
-	rss, isPartial, err := netstorage.ProcessSearchQuery(ec.AuthToken, sq, ec.Deadline)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(ec.AuthToken, sq, true, ec.Deadline)
 	if err != nil {
 		return nil, err
 	}
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 6099b58c8..a627c209e 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -231,10 +231,27 @@ func getMaxPrevInterval(timestamps []int64) int64 {
 	}
 	d := (timestamps[len(timestamps)-1] - timestamps[0]) / int64(len(timestamps)-1)
 	if d <= 0 {
-		return 1
+		return int64(maxSilenceInterval)
+	}
+	// Increase d more for smaller scrape intervals in order to hide possible gaps
+	// when high jitter is present.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/139 .
+	if d <= 2*1000 {
+		return d + 4*d
+	}
+	if d <= 4*1000 {
+		return d + 2*d
+	}
+	if d <= 8*1000 {
+		return d + d
+	}
+	if d <= 16*1000 {
+		return d + d/2
+	}
+	if d <= 32*1000 {
+		return d + d/4
 	}
-	// Slightly increase d in order to handle possible jitter in scrape interval.
-	return d + (d / 4)
+	return d + d/8
 }
 
 func removeCounterResets(values []float64) {
diff --git a/app/vmstorage/transport/server.go b/app/vmstorage/transport/server.go
index 41421e697..a5a2cd6e6 100644
--- a/app/vmstorage/transport/server.go
+++ b/app/vmstorage/transport/server.go
@@ -392,6 +392,18 @@ func (ctx *vmselectRequestCtx) readDataBufBytes(maxDataSize int) error {
 	return nil
 }
 
+func (ctx *vmselectRequestCtx) readBool() (bool, error) {
+	ctx.dataBuf = bytesutil.Resize(ctx.dataBuf, 1)
+	if _, err := io.ReadFull(ctx.bc, ctx.dataBuf); err != nil {
+		if err == io.EOF {
+			return false, err
+		}
+		return false, fmt.Errorf("cannot read bool: %s", err)
+	}
+	v := ctx.dataBuf[0] != 0
+	return v, nil
+}
+
 func (ctx *vmselectRequestCtx) writeDataBufBytes() error {
 	if err := ctx.writeUint64(uint64(len(ctx.dataBuf))); err != nil {
 		return fmt.Errorf("cannot write data size: %s", err)
@@ -443,7 +455,7 @@ func (s *Server) processVMSelectRequest(ctx *vmselectRequestCtx) error {
 	}()
 
 	switch string(ctx.dataBuf) {
-	case "search_v2":
+	case "search_v3":
 		return s.processVMSelectSearchQuery(ctx)
 	case "labelValues":
 		return s.processVMSelectLabelValues(ctx)
@@ -714,6 +726,10 @@ func (s *Server) processVMSelectSearchQuery(ctx *vmselectRequestCtx) error {
 	if len(tail) > 0 {
 		return fmt.Errorf("unexpected non-zero tail left after unmarshaling SearchQuery: (len=%d) %q", len(tail), tail)
 	}
+	fetchData, err := ctx.readBool()
+	if err != nil {
+		return fmt.Errorf("cannot read `fetchData` bool: %s", err)
+	}
 
 	// Setup search.
 	if err := ctx.setupTfss(); err != nil {
@@ -728,7 +744,7 @@ func (s *Server) processVMSelectSearchQuery(ctx *vmselectRequestCtx) error {
 		MinTimestamp: ctx.sq.MinTimestamp,
 		MaxTimestamp: ctx.sq.MaxTimestamp,
 	}
-	ctx.sr.Init(s.storage, ctx.tfss, tr, *maxMetricsPerSearch)
+	ctx.sr.Init(s.storage, ctx.tfss, tr, fetchData, *maxMetricsPerSearch)
 	defer ctx.sr.MustClose()
 	if err := ctx.sr.Error(); err != nil {
 		// Send the error message to vmselect.
diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
index c0fb03726..cc6c0ef1c 100644
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@@ -21,7 +21,7 @@ services:
     restart: always
   vmstorage:
     container_name: vmstorage
-    image: victoriametrics/vmstorage:v1.23.1-cluster
+    image: victoriametrics/vmstorage:v1.24.0-cluster
     ports:
       - 8482:8482
       - 8400:8400
@@ -35,7 +35,7 @@ services:
     restart: always
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.23.1-cluster
+    image: victoriametrics/vminsert:v1.24.0-cluster
     depends_on:
       - "vmstorage"
     command:
@@ -47,7 +47,7 @@ services:
     restart: always
   vmselect:
     container_name: vmselect
-    image: victoriametrics/vmselect:v1.23.1-cluster
+    image: victoriametrics/vmselect:v1.24.0-cluster
     depends_on:
       - "vmstorage"
     command:
diff --git a/deployment/k8s/helm/victoria-metrics/templates/vmselect-deployment.yaml b/deployment/k8s/helm/victoria-metrics/templates/vmselect-deployment.yaml
index 2d987f4b7..c2631e73a 100644
--- a/deployment/k8s/helm/victoria-metrics/templates/vmselect-deployment.yaml
+++ b/deployment/k8s/helm/victoria-metrics/templates/vmselect-deployment.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.vmselect.enabled -}}
+{{- if and .Values.vmselect.enabled (not .Values.vmselect.statefulSet.enabled) -}}
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -68,5 +68,10 @@ spec:
     {{- end }}
       volumes:
         - name: cache-volume
+        {{- if .Values.vmselect.persistentVolume.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.vmselect.persistentVolume.existingClaim }}{{ .Values.vmselect.persistentVolume.existingClaim }}{{- else }}{{ template "victoria-metrics.vmselect.fullname" . }}{{- end }}
+            {{- else }}
           emptyDir: {}
+            {{- end -}}
 {{- end }}
diff --git a/deployment/k8s/helm/victoria-metrics/templates/vmselect-service-headless.yaml b/deployment/k8s/helm/victoria-metrics/templates/vmselect-service-headless.yaml
new file mode 100644
index 000000000..d83c9b8d9
--- /dev/null
+++ b/deployment/k8s/helm/victoria-metrics/templates/vmselect-service-headless.yaml
@@ -0,0 +1,24 @@
+{{- if and .Values.vmselect.enabled .Values.vmselect.statefulSet.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.vmselect.statefulSet.service.annotations }}
+  annotations:
+{{ toYaml .Values.vmselect.statefulSet.service.annotations | indent 4}}
+{{- end }}
+  labels:
+  {{- include "victoria-metrics.vmselect.labels" . | nindent 4 }}
+{{- if .Values.vmselect.statefulSet.service.labels }}
+{{ toYaml .Values.vmselect.statefulSet.service.labels | indent 4}}
+{{- end }}
+  name: {{ template "victoria-metrics.vmselect.fullname" . }}
+spec:
+  clusterIP: None
+  ports:
+    - name: http
+      port: {{ .Values.vmselect.statefulSet.service.servicePort }}
+      protocol: TCP
+      targetPort: http
+  selector:
+    {{- include "victoria-metrics.vmselect.matchLabels" . | nindent 4 }}
+{{- end -}}
diff --git a/deployment/k8s/helm/victoria-metrics/templates/vmselect-service.yaml b/deployment/k8s/helm/victoria-metrics/templates/vmselect-service.yaml
index f92120d44..ca91e2a21 100644
--- a/deployment/k8s/helm/victoria-metrics/templates/vmselect-service.yaml
+++ b/deployment/k8s/helm/victoria-metrics/templates/vmselect-service.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.vmselect.enabled -}}
+{{- if and .Values.vmselect.enabled (not .Values.vmselect.statefulSet.enabled) -}}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/deployment/k8s/helm/victoria-metrics/templates/vmselect-statefulset.yaml b/deployment/k8s/helm/victoria-metrics/templates/vmselect-statefulset.yaml
new file mode 100644
index 000000000..6d71faaff
--- /dev/null
+++ b/deployment/k8s/helm/victoria-metrics/templates/vmselect-statefulset.yaml
@@ -0,0 +1,97 @@
+{{- if and .Values.vmselect.enabled .Values.vmselect.statefulSet.enabled -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    {{- include "victoria-metrics.vmselect.labels" . | nindent 4 }}
+  name: {{ template "victoria-metrics.vmselect.fullname" . }}
+spec:
+  serviceName: {{ template "victoria-metrics.vmselect.fullname" . }}
+  selector:
+    matchLabels:
+      {{- include "victoria-metrics.vmselect.matchLabels" . | nindent 6 }}
+  replicas: {{ .Values.vmselect.replicaCount }}
+  podManagementPolicy: {{ .Values.vmselect.podManagementPolicy }}
+  template:
+    metadata:
+    {{- if .Values.vmselect.podAnnotations }}
+      annotations:
+{{ toYaml .Values.vmselect.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        {{- include "victoria-metrics.vmselect.labels" . | nindent 8 }}
+    spec:
+{{- if .Values.vmselect.priorityClassName }}
+      priorityClassName: "{{ .Values.vmselect.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: {{ template "victoria-metrics.name" . }}-{{ .Values.vmselect.name }}
+          image: "{{ .Values.vmselect.image.repository }}:{{ .Values.vmselect.image.tag }}"
+          imagePullPolicy: "{{ .Values.vmselect.image.pullPolicy }}"
+          args:
+            - {{ printf "%s=%s" "--cacheDataPath" .Values.vmselect.cacheMountPath | quote}}
+          {{- include "victoria-metrics.vmselect.vmstorage-pod-fqdn" . | nindent 12 }}
+          {{- range $key, $value := .Values.vmselect.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: 8481
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 15
+          volumeMounts:
+            - mountPath: {{ .Values.vmselect.cacheMountPath }}
+              name: cache-volume
+          resources:
+{{ toYaml .Values.vmselect.resources | indent 12 }}
+    {{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{ toYaml .Values.imagePullSecrets | indent 2 }}
+    {{- end }}
+    {{- if .Values.vmselect.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.vmselect.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.vmselect.securityContext }}
+      securityContext:
+{{ toYaml .Values.vmselect.securityContext | indent 8 }}
+    {{- end }}
+    {{- if .Values.vmselect.tolerations }}
+      tolerations:
+{{ toYaml .Values.vmselect.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.vmselect.affinity }}
+      affinity:
+{{ toYaml .Values.vmselect.affinity | indent 8 }}
+    {{- end }}
+{{- if .Values.vmselect.persistentVolume.enabled }}
+  volumeClaimTemplates:
+    - metadata:
+        name: cache-volume
+        {{- if .Values.vmselect.persistentVolume.annotations }}
+        annotations:
+    {{ toYaml .Values.vmselect.persistentVolume.annotations | indent 10 }}
+    {{- end }}
+      spec:
+        accessModes:
+        {{ toYaml .Values.vmselect.persistentVolume.accessModes | indent 10 }}
+        resources:
+          requests:
+            storage: "{{ .Values.vmselect.persistentVolume.size }}"
+          {{- if .Values.vmselect.persistentVolume.storageClass }}
+        {{- if (eq "-" .Values.vmselect.persistentVolume.storageClass) }}
+        storageClassName: ""
+        {{- else }}
+        storageClassName: "{{ .Values.vmselect.persistentVolume.storageClass }}"
+        {{- end }}
+        {{- end }}
+    {{- else }}
+      volumes:
+        - name: cache-volume
+          emptyDir: {}
+{{- end }}
+{{- end }}
diff --git a/deployment/k8s/helm/victoria-metrics/templates/vmstorage-statefulset.yaml b/deployment/k8s/helm/victoria-metrics/templates/vmstorage-statefulset.yaml
index ceef9ee75..3de023bf1 100644
--- a/deployment/k8s/helm/victoria-metrics/templates/vmstorage-statefulset.yaml
+++ b/deployment/k8s/helm/victoria-metrics/templates/vmstorage-statefulset.yaml
@@ -158,6 +158,7 @@ spec:
       {{- end }}
       {{- end }}
 {{- else }}
+      volumes:
         - name: vmstorage-volume
           emptyDir: {}
 {{- end }}
diff --git a/deployment/k8s/helm/victoria-metrics/values.yaml b/deployment/k8s/helm/victoria-metrics/values.yaml
index ce3f59aab..5e5aa2c5a 100644
--- a/deployment/k8s/helm/victoria-metrics/values.yaml
+++ b/deployment/k8s/helm/victoria-metrics/values.yaml
@@ -12,7 +12,7 @@ vmselect:
   name: vmselect
   image:
     repository: victoriametrics/vmselect
-    tag: v1.23.1-cluster
+    tag: v1.24.0-cluster
     pullPolicy: IfNotPresent
   priorityClassName: ""
   extraArgs: {}
@@ -53,13 +53,43 @@ vmselect:
     loadBalancerSourceRanges: []
     servicePort: 8481
     type: ClusterIP
+  statefulSet:
+    ## Creates statefulset instead of deployment, useful when you want to keep the cache
+    enabled: false
+    podManagementPolicy: OrderedReady
+    ## Headless service for statefulset
+    service:
+      annotations: {}
+      labels: {}
+      servicePort: 8481
+  persistentVolume:
+    ## If true, vmselect will create/use a Persistent Volume Claim
+    ## If false, use emptyDir
+    ##
+    enabled: false
+
+    ## Must match those of existing PV or dynamic provisioner
+    ## Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+    ##
+    accessModes:
+      - ReadWriteOnce
+    annotations: {}
+
+    ## Requires vmselect.persistentVolume.enabled: true
+    ## If defined, PVC must be created manually before volume will be bound
+    existingClaim: ""
+
+    ## Vmselect data Persistent Volume mount root path
+    ##
+    size: 2Gi
+    subPath: ""
 
 vminsert:
   enabled: true
   name: vminsert
   image:
     repository: victoriametrics/vminsert
-    tag: v1.23.1-cluster
+    tag: v1.24.0-cluster
     pullPolicy: IfNotPresent
   priorityClassName: ""
   extraArgs: {}
@@ -103,7 +133,7 @@ vmstorage:
   name: vmstorage
   image:
     repository: victoriametrics/vmstorage
-    tag: v1.23.1-cluster
+    tag: v1.24.0-cluster
     pullPolicy: IfNotPresent
   priorityClassName: ""
   fullnameOverride:
diff --git a/lib/storage/part_search.go b/lib/storage/part_search.go
index cfbac8cf0..b25fe59a0 100644
--- a/lib/storage/part_search.go
+++ b/lib/storage/part_search.go
@@ -28,6 +28,9 @@ type partSearch struct {
 	// tr is a time range to search.
 	tr TimeRange
 
+	// Skip populating timestampsData and valuesData in Block if fetchData=false.
+	fetchData bool
+
 	metaindex []metaindexRow
 
 	ibCache *indexBlockCache
@@ -61,7 +64,7 @@ func (ps *partSearch) reset() {
 }
 
 // Init initializes the ps with the given p, tsids and tr.
-func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange) {
+func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange, fetchData bool) {
 	ps.reset()
 	ps.p = p
 
@@ -72,6 +75,7 @@ func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange) {
 		ps.tsids = append(ps.tsids[:0], tsids...)
 	}
 	ps.tr = tr
+	ps.fetchData = fetchData
 	ps.metaindex = p.metaindex
 	ps.ibCache = &p.ibCache
 
@@ -281,11 +285,14 @@ func (ps *partSearch) searchBHS() bool {
 
 func (ps *partSearch) readBlock(bh *blockHeader) {
 	ps.Block.Reset()
+	ps.Block.bh = *bh
+	if !ps.fetchData {
+		return
+	}
+
 	ps.Block.timestampsData = bytesutil.Resize(ps.Block.timestampsData[:0], int(bh.TimestampsBlockSize))
 	ps.p.timestampsFile.ReadAt(ps.Block.timestampsData, int64(bh.TimestampsBlockOffset))
 
 	ps.Block.valuesData = bytesutil.Resize(ps.Block.valuesData[:0], int(bh.ValuesBlockSize))
 	ps.p.valuesFile.ReadAt(ps.Block.valuesData, int64(bh.ValuesBlockOffset))
-
-	ps.Block.bh = *bh
 }
diff --git a/lib/storage/partition_search.go b/lib/storage/partition_search.go
index 815ec3589..475ffed5a 100644
--- a/lib/storage/partition_search.go
+++ b/lib/storage/partition_search.go
@@ -56,7 +56,7 @@ func (pts *partitionSearch) reset() {
 // Init initializes the search in the given partition for the given tsid and tr.
 //
 // MustClose must be called when partition search is done.
-func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange) {
+func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange, fetchData bool) {
 	if pts.needClosing {
 		logger.Panicf("BUG: missing partitionSearch.MustClose call before the next call to Init")
 	}
@@ -85,7 +85,7 @@ func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange) {
 	}
 	pts.psPool = pts.psPool[:len(pts.pws)]
 	for i, pw := range pts.pws {
-		pts.psPool[i].Init(pw.p, tsids, tr)
+		pts.psPool[i].Init(pw.p, tsids, tr, fetchData)
 	}
 
 	// Initialize the psHeap.
diff --git a/lib/storage/search.go b/lib/storage/search.go
index 468824f28..0433c8aa3 100644
--- a/lib/storage/search.go
+++ b/lib/storage/search.go
@@ -110,7 +110,7 @@ func (s *Search) reset() {
 // Init initializes s from the given storage, tfss and tr.
 //
 // MustClose must be called when the search is done.
-func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, maxMetrics int) {
+func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, fetchData bool, maxMetrics int) {
 	if s.needClosing {
 		logger.Panicf("BUG: missing MustClose call before the next call to Init")
 	}
@@ -123,7 +123,7 @@ func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, maxMet
 	// It is ok to call Init on error from storage.searchTSIDs.
 	// Init must be called before returning because it will fail
 	// on Seach.MustClose otherwise.
-	s.ts.Init(storage.tb, tsids, tr)
+	s.ts.Init(storage.tb, tsids, tr, fetchData)
 
 	if err != nil {
 		s.err = err
diff --git a/lib/storage/table_search.go b/lib/storage/table_search.go
index 8a9c31129..aa5b81476 100644
--- a/lib/storage/table_search.go
+++ b/lib/storage/table_search.go
@@ -55,7 +55,7 @@ func (ts *tableSearch) reset() {
 // Init initializes the ts.
 //
 // MustClose must be called then the tableSearch is done.
-func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange) {
+func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange, fetchData bool) {
 	if ts.needClosing {
 		logger.Panicf("BUG: missing MustClose call before the next call to Init")
 	}
@@ -86,7 +86,7 @@ func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange) {
 	}
 	ts.ptsPool = ts.ptsPool[:len(ts.ptws)]
 	for i, ptw := range ts.ptws {
-		ts.ptsPool[i].Init(ptw.pt, tsids, tr)
+		ts.ptsPool[i].Init(ptw.pt, tsids, tr, fetchData)
 	}
 
 	// Initialize the ptsHeap.


Test patch:
diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go
index fdb79c8d3..fcf598b63 100644
--- a/app/vmselect/promql/rollup_test.go
+++ b/app/vmselect/promql/rollup_test.go
@@ -347,7 +347,7 @@ func TestRollupNoWindowNoPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{2, 0, 0, 0, 0, 0, 0, nan}
+		valuesExpected := []float64{2, 0, 0, 0, 0, 0, 0, 0}
 		timestampsExpected := []int64{120, 124, 128, 132, 136, 140, 144, 148}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -371,15 +371,15 @@ func TestRollupWindowNoPoints(t *testing.T) {
 	t.Run("afterEnd", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupFirst,
-			Start:  141,
-			End:    171,
+			Start:  161,
+			End:    191,
 			Step:   10,
 			Window: 3,
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{34, nan, nan, nan}
-		timestampsExpected := []int64{141, 151, 161, 171}
+		valuesExpected := []float64{34, 34, 34, nan}
+		timestampsExpected := []int64{161, 171, 181, 191}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
diff --git a/lib/storage/part_search_test.go b/lib/storage/part_search_test.go
index a56f04f01..11166debf 100644
--- a/lib/storage/part_search_test.go
+++ b/lib/storage/part_search_test.go
@@ -1247,7 +1247,7 @@ func testPartSearch(t *testing.T, p *part, tsids []TSID, tr TimeRange, expectedR
 
 func testPartSearchSerial(p *part, tsids []TSID, tr TimeRange, expectedRawBlocks []rawBlock) error {
 	var ps partSearch
-	ps.Init(p, tsids, tr)
+	ps.Init(p, tsids, tr, true)
 	var bs []Block
 	for ps.NextBlock() {
 		var b Block
diff --git a/lib/storage/partition_search_test.go b/lib/storage/partition_search_test.go
index 0b23ecc16..fba878590 100644
--- a/lib/storage/partition_search_test.go
+++ b/lib/storage/partition_search_test.go
@@ -238,7 +238,7 @@ func testPartitionSearchSerial(pt *partition, tsids []TSID, tr TimeRange, rbsExp
 
 	bs := []Block{}
 	var pts partitionSearch
-	pts.Init(pt, tsids, tr)
+	pts.Init(pt, tsids, tr, true)
 	for pts.NextBlock() {
 		var b Block
 		b.CopyFrom(pts.Block)
@@ -263,7 +263,7 @@ func testPartitionSearchSerial(pt *partition, tsids []TSID, tr TimeRange, rbsExp
 	}
 
 	// verify that empty tsids returns empty result
-	pts.Init(pt, []TSID{}, tr)
+	pts.Init(pt, []TSID{}, tr, true)
 	if pts.NextBlock() {
 		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", pts.Block)
 	}
@@ -271,6 +271,16 @@ func testPartitionSearchSerial(pt *partition, tsids []TSID, tr TimeRange, rbsExp
 		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
 	}
 	pts.MustClose()
+
+	pts.Init(pt, []TSID{}, tr, false)
+	if pts.NextBlock() {
+		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", pts.Block)
+	}
+	if err := pts.Error(); err != nil {
+		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
+	}
+	pts.MustClose()
+
 	return nil
 }
 
diff --git a/lib/storage/search_test.go b/lib/storage/search_test.go
index 69c9512c3..924baf845 100644
--- a/lib/storage/search_test.go
+++ b/lib/storage/search_test.go
@@ -200,7 +200,7 @@ func testSearch(st *Storage, tr TimeRange, mrs []MetricRow, accountsCount int) e
 		}
 
 		// Search
-		s.Init(st, []*TagFilters{tfs}, tr, 1e5)
+		s.Init(st, []*TagFilters{tfs}, tr, true, 1e5)
 		var mbs []MetricBlock
 		for s.NextMetricBlock() {
 			var b Block
diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go
index a73684eec..958fca790 100644
--- a/lib/storage/storage_test.go
+++ b/lib/storage/storage_test.go
@@ -506,12 +506,24 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error {
 		MaxTimestamp: 2e10,
 	}
 	metricBlocksCount := func(tfs *TagFilters) int {
+		// Verify the number of blocks with fetchData=true
 		n := 0
-		sr.Init(s, []*TagFilters{tfs}, tr, 1e5)
+		sr.Init(s, []*TagFilters{tfs}, tr, true, 1e5)
 		for sr.NextMetricBlock() {
 			n++
 		}
 		sr.MustClose()
+
+		// Make sure the number of blocks with fetchData=false is the same.
+		m := 0
+		sr.Init(s, []*TagFilters{tfs}, tr, false, 1e5)
+		for sr.NextMetricBlock() {
+			m++
+		}
+		sr.MustClose()
+		if n != m {
+			return -1
+		}
 		return n
 	}
 	for i := 0; i < metricsCount; i++ {
diff --git a/lib/storage/table_search_test.go b/lib/storage/table_search_test.go
index c116e4e84..6c75a1a08 100644
--- a/lib/storage/table_search_test.go
+++ b/lib/storage/table_search_test.go
@@ -251,7 +251,7 @@ func testTableSearchSerial(tb *table, tsids []TSID, tr TimeRange, rbsExpected []
 
 	bs := []Block{}
 	var ts tableSearch
-	ts.Init(tb, tsids, tr)
+	ts.Init(tb, tsids, tr, true)
 	for ts.NextBlock() {
 		var b Block
 		b.CopyFrom(ts.Block)
@@ -276,7 +276,7 @@ func testTableSearchSerial(tb *table, tsids []TSID, tr TimeRange, rbsExpected []
 	}
 
 	// verify that empty tsids returns empty result
-	ts.Init(tb, []TSID{}, tr)
+	ts.Init(tb, []TSID{}, tr, true)
 	if ts.NextBlock() {
 		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", ts.Block)
 	}
@@ -284,5 +284,15 @@ func testTableSearchSerial(tb *table, tsids []TSID, tr TimeRange, rbsExpected []
 		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
 	}
 	ts.MustClose()
+
+	ts.Init(tb, []TSID{}, tr, false)
+	if ts.NextBlock() {
+		return fmt.Errorf("unexpected block got for an empty tsids list with fetchData=false: %+v", ts.Block)
+	}
+	if err := ts.Error(); err != nil {
+		return fmt.Errorf("unexpected error on empty tsids list with fetchData=false: %s", err)
+	}
+	ts.MustClose()
+
 	return nil
 }
diff --git a/lib/storage/table_search_timing_test.go b/lib/storage/table_search_timing_test.go
index 917664de6..df9af44f4 100644
--- a/lib/storage/table_search_timing_test.go
+++ b/lib/storage/table_search_timing_test.go
@@ -26,7 +26,11 @@ func BenchmarkTableSearch(b *testing.B) {
 				b.Run(fmt.Sprintf("tsidsCount_%d", tsidsCount), func(b *testing.B) {
 					for _, tsidsSearch := range []int{1, 1e1, 1e2, 1e3, 1e4} {
 						b.Run(fmt.Sprintf("tsidsSearch_%d", tsidsSearch), func(b *testing.B) {
-							benchmarkTableSearch(b, rowsCount, tsidsCount, tsidsSearch)
+							for _, fetchData := range []bool{true, false} {
+								b.Run(fmt.Sprintf("fetchData_%v", fetchData), func(b *testing.B) {
+									benchmarkTableSearch(b, rowsCount, tsidsCount, tsidsSearch, fetchData)
+								})
+							}
 						})
 					}
 				})
@@ -103,9 +107,9 @@ func createBenchTable(b *testing.B, path string, startTimestamp int64, rowsPerIn
 	tb.MustClose()
 }
 
-func benchmarkTableSearch(b *testing.B, rowsCount, tsidsCount, tsidsSearch int) {
+func benchmarkTableSearch(b *testing.B, rowsCount, tsidsCount, tsidsSearch int, fetchData bool) {
 	startTimestamp := timestampFromTime(time.Now()) - 365*24*3600*1000
-	rowsPerInsert := int(maxRawRowsPerPartition)
+	rowsPerInsert := getMaxRawRowsPerPartition()
 
 	tb := openBenchTable(b, startTimestamp, rowsPerInsert, rowsCount, tsidsCount)
 	tr := TimeRange{
@@ -127,7 +131,7 @@ func benchmarkTableSearch(b *testing.B, rowsCount, tsidsCount, tsidsSearch int)
 			for i := range tsids {
 				tsids[i].MetricID = 1 + uint64(i)
 			}
-			ts.Init(tb, tsids, tr)
+			ts.Init(tb, tsids, tr, fetchData)
 			for ts.NextBlock() {
 			}
 			ts.MustClose()