Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4021 (2023-03-27 19:32:09)

app/vmselect: Fix le buckets when adjacent vmrange is empty
There is a bug here where if you have a single bucket like:

`foo{vmrange="4.084e+02...4.642e+02"} 2 123`

The expected output is three le encoded buckets like:

```
foo{le="4.084e+02"} 0 123
foo{le="4.642e+02"} 2 123
foo{le="+Inf"} 2 123
```

This correctly encodes the start and end of the vmrange. If however, the input contains the previous bucket, and that bucket is empty then you only get the end le and +Inf out currently, i.e:

```
foo{vmrange="7.743e+05...8.799e+05"} 5 123
foo{vmrange="6.813e+05...7.743e+05"} 0 123
```

results in:

```
foo{le="8.799e+05"} 5 123
foo{le="+Inf"} 5 123
```

This causes issues when you go to compute a quantile because this means that the assumed lower bound of the buckets is 0 and this we interpolate between 0->end rather than the vmrange start->end as expected.

I am not in love with the structure of this fix... but there are a ton of special cases here and this was the simplest thing I could find that both preserved existing behavior while fixing the issue at hand. Suggestions welcome of course!


Patch:
diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go
index 3cde8a7c7..0b584a94d 100644
--- a/app/vmalert/web.qtpl.go
+++ b/app/vmalert/web.qtpl.go
@@ -227,7 +227,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 //line app/vmalert/web.qtpl:55
 			qw422016.N().FPrec(g.Interval, 0)
 //line app/vmalert/web.qtpl:55
-			qw422016.N().S(`s)  #</a>
+			qw422016.N().S(`s) #</a>
                  `)
 //line app/vmalert/web.qtpl:56
 			if rNotOk[g.ID] > 0 {
diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index 1c91a4853..da71e71ce 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -747,7 +747,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	} else {
 		queryOffset = 0
 	}
-	ec := promql.EvalConfig{
+	qs := &promql.QueryStats{}
+	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 start,
 		Step:                step,
@@ -762,8 +763,10 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
+
+		QueryStats: qs,
 	}
-	result, err := promql.Exec(qt, &ec, query, true)
+	result, err := promql.Exec(qt, ec, query, true)
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
 	}
@@ -786,7 +789,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	qtDone := func() {
 		qt.Donef("query=%s, time=%d: series=%d", query, start, len(result))
 	}
-	WriteQueryResponse(bw, result, qt, qtDone)
+
+	WriteQueryResponse(bw, result, qt, qtDone, qs)
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
@@ -851,7 +855,8 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		start, end = promql.AdjustStartEnd(start, end, step)
 	}
 
-	ec := promql.EvalConfig{
+	qs := &promql.QueryStats{}
+	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 end,
 		Step:                step,
@@ -866,8 +871,10 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
+
+		QueryStats: qs,
 	}
-	result, err := promql.Exec(qt, &ec, query, false)
+	result, err := promql.Exec(qt, ec, query, false)
 	if err != nil {
 		return err
 	}
@@ -891,7 +898,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	qtDone := func() {
 		qt.Donef("start=%d, end=%d, step=%d, query=%q: series=%d", start, end, step, query, len(result))
 	}
-	WriteQueryRangeResponse(bw, result, qt, qtDone)
+	WriteQueryRangeResponse(bw, result, qt, qtDone, qs)
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}
diff --git a/app/vmselect/prometheus/query_range_response.qtpl b/app/vmselect/prometheus/query_range_response.qtpl
index 1d5b74819..861903d80 100644
--- a/app/vmselect/prometheus/query_range_response.qtpl
+++ b/app/vmselect/prometheus/query_range_response.qtpl
@@ -1,12 +1,13 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryRangeResponse generates response for /api/v1/query_range.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-{% func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
+{% func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %}
 {
 	{% code
 		seriesCount := len(rs)
@@ -26,6 +27,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 				{% endfor %}
 			{% endif %}
 		]
+	},
+	"stats":{
+	    "seriesFetched": "{%d qs.SeriesFetched %}"
 	}
 	{% code
 		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)
diff --git a/app/vmselect/prometheus/query_range_response.qtpl.go b/app/vmselect/prometheus/query_range_response.qtpl.go
index ac8dd6f23..68451941a 100644
--- a/app/vmselect/prometheus/query_range_response.qtpl.go
+++ b/app/vmselect/prometheus/query_range_response.qtpl.go
@@ -7,133 +7,138 @@ package prometheus
 //line app/vmselect/prometheus/query_range_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryRangeResponse generates response for /api/v1/query_range.See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
-func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
+func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 	qw422016.N().S(`{`)
-//line app/vmselect/prometheus/query_range_response.qtpl:12
+//line app/vmselect/prometheus/query_range_response.qtpl:13
 	seriesCount := len(rs)
 	pointsCount := 0
 
-//line app/vmselect/prometheus/query_range_response.qtpl:14
+//line app/vmselect/prometheus/query_range_response.qtpl:15
 	qw422016.N().S(`"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vmselect/prometheus/query_range_response.qtpl:19
-	if len(rs) > 0 {
 //line app/vmselect/prometheus/query_range_response.qtpl:20
-		streamqueryRangeLine(qw422016, &rs[0])
+	if len(rs) > 0 {
 //line app/vmselect/prometheus/query_range_response.qtpl:21
+		streamqueryRangeLine(qw422016, &rs[0])
+//line app/vmselect/prometheus/query_range_response.qtpl:22
 		pointsCount += len(rs[0].Values)
 
-//line app/vmselect/prometheus/query_range_response.qtpl:22
+//line app/vmselect/prometheus/query_range_response.qtpl:23
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_range_response.qtpl:23
+//line app/vmselect/prometheus/query_range_response.qtpl:24
 		for i := range rs {
-//line app/vmselect/prometheus/query_range_response.qtpl:23
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/query_range_response.qtpl:24
-			streamqueryRangeLine(qw422016, &rs[i])
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/query_range_response.qtpl:25
+			streamqueryRangeLine(qw422016, &rs[i])
+//line app/vmselect/prometheus/query_range_response.qtpl:26
 			pointsCount += len(rs[i].Values)
 
-//line app/vmselect/prometheus/query_range_response.qtpl:26
-		}
 //line app/vmselect/prometheus/query_range_response.qtpl:27
+		}
+//line app/vmselect/prometheus/query_range_response.qtpl:28
 	}
-//line app/vmselect/prometheus/query_range_response.qtpl:27
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:28
+	qw422016.N().S(`]},"stats":{"seriesFetched": "`)
+//line app/vmselect/prometheus/query_range_response.qtpl:32
+	qw422016.N().D(qs.SeriesFetched)
+//line app/vmselect/prometheus/query_range_response.qtpl:32
+	qw422016.N().S(`"}`)
+//line app/vmselect/prometheus/query_range_response.qtpl:35
 	qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)
 	qtDone()
 
-//line app/vmselect/prometheus/query_range_response.qtpl:34
+//line app/vmselect/prometheus/query_range_response.qtpl:38
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/query_range_response.qtpl:34
+//line app/vmselect/prometheus/query_range_response.qtpl:38
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	StreamQueryRangeResponse(qw422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	StreamQueryRangeResponse(qw422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string {
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	WriteQueryRangeResponse(qb422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	WriteQueryRangeResponse(qb422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:36
+//line app/vmselect/prometheus/query_range_response.qtpl:40
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:38
+//line app/vmselect/prometheus/query_range_response.qtpl:42
 func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:38
+//line app/vmselect/prometheus/query_range_response.qtpl:42
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:40
+//line app/vmselect/prometheus/query_range_response.qtpl:44
 	streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_range_response.qtpl:40
+//line app/vmselect/prometheus/query_range_response.qtpl:44
 	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:41
+//line app/vmselect/prometheus/query_range_response.qtpl:45
 	streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps)
-//line app/vmselect/prometheus/query_range_response.qtpl:41
+//line app/vmselect/prometheus/query_range_response.qtpl:45
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	streamqueryRangeLine(qw422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 func queryRangeLine(r *netstorage.Result) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	writequeryRangeLine(qb422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }
diff --git a/app/vmselect/prometheus/query_response.qtpl b/app/vmselect/prometheus/query_response.qtpl
index 4f3e676c7..590d73011 100644
--- a/app/vmselect/prometheus/query_response.qtpl
+++ b/app/vmselect/prometheus/query_response.qtpl
@@ -1,12 +1,13 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryResponse generates response for /api/v1/query.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
+{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %}
 {
 	{% code seriesCount := len(rs) %}
 	"status":"success",
@@ -28,6 +29,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 				{% endfor %}
 			{% endif %}
 		]
+	},
+	"stats":{
+	    "seriesFetched": "{%d qs.SeriesFetched %}"
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
diff --git a/app/vmselect/prometheus/query_response.qtpl.go b/app/vmselect/prometheus/query_response.qtpl.go
index 51fa51188..9a21336c5 100644
--- a/app/vmselect/prometheus/query_response.qtpl.go
+++ b/app/vmselect/prometheus/query_response.qtpl.go
@@ -7,102 +7,107 @@ package prometheus
 //line app/vmselect/prometheus/query_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryResponse generates response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:9
-func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
+func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_response.qtpl:10
 	qw422016.N().S(`{`)
-//line app/vmselect/prometheus/query_response.qtpl:11
+//line app/vmselect/prometheus/query_response.qtpl:12
 	seriesCount := len(rs)
 
-//line app/vmselect/prometheus/query_response.qtpl:11
+//line app/vmselect/prometheus/query_response.qtpl:12
 	qw422016.N().S(`"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:17
 	if len(rs) > 0 {
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:17
 		qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:18
+//line app/vmselect/prometheus/query_response.qtpl:19
 		streammetricNameObject(qw422016, &rs[0].MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:18
-		qw422016.N().S(`,"value":`)
 //line app/vmselect/prometheus/query_response.qtpl:19
+		qw422016.N().S(`,"value":`)
+//line app/vmselect/prometheus/query_response.qtpl:20
 		streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:19
+//line app/vmselect/prometheus/query_response.qtpl:20
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:21
+//line app/vmselect/prometheus/query_response.qtpl:22
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_response.qtpl:22
-		for i := range rs {
 //line app/vmselect/prometheus/query_response.qtpl:23
+		for i := range rs {
+//line app/vmselect/prometheus/query_response.qtpl:24
 			r := &rs[i]
 
-//line app/vmselect/prometheus/query_response.qtpl:23
+//line app/vmselect/prometheus/query_response.qtpl:24
 			qw422016.N().S(`,{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:25
+//line app/vmselect/prometheus/query_response.qtpl:26
 			streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:25
-			qw422016.N().S(`,"value":`)
 //line app/vmselect/prometheus/query_response.qtpl:26
+			qw422016.N().S(`,"value":`)
+//line app/vmselect/prometheus/query_response.qtpl:27
 			streammetricRow(qw422016, r.Timestamps[0], r.Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:26
+//line app/vmselect/prometheus/query_response.qtpl:27
 			qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:28
-		}
 //line app/vmselect/prometheus/query_response.qtpl:29
+		}
+//line app/vmselect/prometheus/query_response.qtpl:30
 	}
-//line app/vmselect/prometheus/query_response.qtpl:29
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/query_response.qtpl:33
+//line app/vmselect/prometheus/query_response.qtpl:30
+	qw422016.N().S(`]},"stats":{"seriesFetched": "`)
+//line app/vmselect/prometheus/query_response.qtpl:34
+	qw422016.N().D(qs.SeriesFetched)
+//line app/vmselect/prometheus/query_response.qtpl:34
+	qw422016.N().S(`"}`)
+//line app/vmselect/prometheus/query_response.qtpl:37
 	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
 	qtDone()
 
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:40
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:40
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
-func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
-	StreamQueryResponse(qw422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+	StreamQueryResponse(qw422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
-func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string {
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_response.qtpl:38
-	WriteQueryResponse(qb422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+	WriteQueryResponse(qb422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 	return qs422016
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }
diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go
index 934d33c63..2ced963ce 100644
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -132,6 +132,12 @@ type EvalConfig struct {
 	// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
 	GetRequestURI func() string
 
+	// QueryStats contains various stats for the currently executed query.
+	//
+	// The caller must initialize the QueryStats if it needs the stats.
+	// Otherwise the stats isn't collected.
+	QueryStats *QueryStats
+
 	timestamps     []int64
 	timestampsOnce sync.Once
 }
@@ -150,11 +156,24 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
 	ec.GetRequestURI = src.GetRequestURI
+	ec.QueryStats = src.QueryStats
 
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
 }
 
+// QueryStats contains various stats for the query.
+type QueryStats struct {
+	// SeriesFetched contains the number of series fetched from storage during the query evaluation.
+	SeriesFetched int
+}
+
+func (qs *QueryStats) addSeriesFetched(n int) {
+	if qs != nil {
+		qs.SeriesFetched += n
+	}
+}
+
 func (ec *EvalConfig) validate() {
 	if ec.Start > ec.End {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@@ -1077,6 +1096,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 		tss := mergeTimeseries(tssCached, nil, start, ec)
 		return tss, nil
 	}
+	ec.QueryStats.addSeriesFetched(rssLen)
 
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index 1197d59aa..279d2e6c3 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -552,14 +552,25 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 		sort.Slice(xss, func(i, j int) bool { return xss[i].end < xss[j].end })
 		xssNew := make([]x, 0, len(xss)+2)
 		var xsPrev x
+		hasNonEmpty := false
 		uniqTs := make(map[string]*timeseries, len(xss))
 		for _, xs := range xss {
 			ts := xs.ts
 			if isZeroTS(ts) {
-				// Skip time series with zeros. They are substituted by xssNew below.
 				xsPrev = xs
+
+				if uniqTs[xs.endStr] == nil {
+					uniqTs[xs.endStr] = xs.ts
+					xssNew = append(xssNew, x{
+						endStr: xs.endStr,
+						end:    xs.end,
+						ts:     copyTS(ts, xs.endStr),
+					})
+				}
 				continue
 			}
+
+			hasNonEmpty = true
 			if xs.start != xsPrev.end && uniqTs[xs.startStr] == nil {
 				uniqTs[xs.startStr] = xs.ts
 				xssNew = append(xssNew, x{
@@ -579,6 +590,12 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 			}
 			xsPrev = xs
 		}
+
+		if !hasNonEmpty {
+			xssNew = []x{}
+			continue
+		}
+
 		if !math.IsInf(xsPrev.end, 1) && !isZeroTS(xsPrev.ts) {
 			xssNew = append(xssNew, x{
 				endStr: "+Inf",
diff --git a/go.mod b/go.mod
index 32e6d58bb..25a5f2526 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/influxdata/influxdb v1.11.0
 	github.com/klauspost/compress v1.16.3
 	github.com/prometheus/prometheus v0.43.0
-	github.com/urfave/cli/v2 v2.25.0
+	github.com/urfave/cli/v2 v2.25.1
 	github.com/valyala/fastjson v1.6.4
 	github.com/valyala/fastrand v1.1.0
 	github.com/valyala/fasttemplate v1.2.2
@@ -47,7 +47,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/aws/aws-sdk-go v1.44.229 // indirect
+	github.com/aws/aws-sdk-go v1.44.230 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect
@@ -112,7 +112,7 @@ require (
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 // indirect
+	google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 // indirect
 	google.golang.org/grpc v1.54.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index 615c0c95a..fe8771680 100644
--- a/go.sum
+++ b/go.sum
@@ -86,8 +86,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
 github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.44.229 h1:lku0ZSHRzj/qtFVM//QE8VjV6kvJ6CFijDZSsjNaD9A=
-github.com/aws/aws-sdk-go v1.44.229/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.230 h1:dcn7TjLyx/31I+0XytMGYRxDc756BRUzsSYVcSyKZlk=
+github.com/aws/aws-sdk-go v1.44.230/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg=
 github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
@@ -414,8 +414,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8=
-github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw=
+github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
@@ -729,8 +729,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 h1:VmCWItVXcKboEMCwZaWge+1JLiTCQSngZeINF+wzO+g=
-google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
+google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 h1:Kd6tRRHXw8z4TlPlWi+NaK10gsePL6GdZBQChptOLGA=
+google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go
index 55e65f655..e34ec16cb 100644
--- a/vendor/github.com/aws/aws-sdk-go/aws/version.go
+++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go
@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.44.229"
+const SDKVersion = "1.44.230"
diff --git a/vendor/github.com/urfave/cli/v2/command.go b/vendor/github.com/urfave/cli/v2/command.go
index f978b4a43..69a0fdf6d 100644
--- a/vendor/github.com/urfave/cli/v2/command.go
+++ b/vendor/github.com/urfave/cli/v2/command.go
@@ -135,6 +135,7 @@ func (c *Command) setup(ctx *Context) {
 		if scmd.HelpName == "" {
 			scmd.HelpName = fmt.Sprintf("%s %s", c.HelpName, scmd.Name)
 		}
+		scmd.separator = c.separator
 		newCmds = append(newCmds, scmd)
 	}
 	c.Subcommands = newCmds
diff --git a/vendor/modules.txt b/vendor/modules.txt
index c37ec5038..1f62cc1d2 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -81,7 +81,7 @@ github.com/VividCortex/ewma
 # github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 ## explicit; go 1.15
 github.com/alecthomas/units
-# github.com/aws/aws-sdk-go v1.44.229
+# github.com/aws/aws-sdk-go v1.44.230
 ## explicit; go 1.11
 github.com/aws/aws-sdk-go/aws
 github.com/aws/aws-sdk-go/aws/awserr
@@ -445,7 +445,7 @@ github.com/russross/blackfriday/v2
 ## explicit; go 1.13
 github.com/stretchr/testify/assert
 github.com/stretchr/testify/require
-# github.com/urfave/cli/v2 v2.25.0
+# github.com/urfave/cli/v2 v2.25.1
 ## explicit; go 1.18
 github.com/urfave/cli/v2
 # github.com/valyala/bytebufferpool v1.0.0
@@ -604,7 +604,7 @@ google.golang.org/appengine/internal/socket
 google.golang.org/appengine/internal/urlfetch
 google.golang.org/appengine/socket
 google.golang.org/appengine/urlfetch
-# google.golang.org/genproto v0.0.0-20230323212658-478b75c54725
+# google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5
 ## explicit; go 1.19
 google.golang.org/genproto/googleapis/api
 google.golang.org/genproto/googleapis/api/annotations


Test patch:
diff --git a/app/vmselect/promql/eval_test.go b/app/vmselect/promql/eval_test.go
index 76c735b21..46e54eda1 100644
--- a/app/vmselect/promql/eval_test.go
+++ b/app/vmselect/promql/eval_test.go
@@ -76,3 +76,21 @@ func TestValidateMaxPointsPerSeriesSuccess(t *testing.T) {
 	f(1659962171908, 1659966077742, 5000, 800)
 	f(1659962150000, 1659966070000, 10000, 393)
 }
+
+func TestQueryStats_addSeriesFetched(t *testing.T) {
+	qs := &QueryStats{}
+	ec := &EvalConfig{
+		QueryStats: qs,
+	}
+	ec.QueryStats.addSeriesFetched(1)
+
+	if qs.SeriesFetched != 1 {
+		t.Fatalf("expected to get 1; got %d instead", qs.SeriesFetched)
+	}
+
+	ecNew := copyEvalConfig(ec)
+	ecNew.QueryStats.addSeriesFetched(3)
+	if qs.SeriesFetched != 4 {
+		t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched)
+	}
+}
diff --git a/app/vmselect/promql/transform_test.go b/app/vmselect/promql/transform_test.go
index 5c5d7e414..6b9b8c476 100644
--- a/app/vmselect/promql/transform_test.go
+++ b/app/vmselect/promql/transform_test.go
@@ -78,6 +78,15 @@ foo{le="+Inf"} 1.23 456`,
 foo{le="+Inf"} 5.3 0`,
 	)
 
+	// Adjacent empty vmrange bucket
+	f(
+		`foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+
 	// Multiple non-empty vmrange buckets
 	f(
 		`foo{vmrange="4.084e+02...4.642e+02"} 2 123