Repository: VictoriaMetrics/VictoriaMetrics
Pull request #961 (2020-12-14 07:37:42)

Changes targets api

codecov[bot]: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=h1) Report
> Merging [#961](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=desc) (eefa797) into [master](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/fd9fd191b91ef4c9965ecb0a73ee7dd0161c8d3e?el=desc) (fd9fd19) will **decrease** coverage by `0.25%`.
> The diff coverage is `0.00%`.

[![Impacted file tree graph](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/graphs/tree.svg?width=650&height=150&src=pr&token=YND8dYVcXn)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #961      +/-   ##
==========================================
- Coverage   60.89%   60.64%   -0.26%     
==========================================
  Files         201      202       +1     
  Lines       22696    22757      +61     
==========================================
- Hits        13821    13801      -20     
- Misses       7557     7647      +90     
+ Partials     1318     1309       -9     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [app/victoria-metrics/main.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-YXBwL3ZpY3RvcmlhLW1ldHJpY3MvbWFpbi5nbw==) | `7.27% <0.00%> (-2.03%)` | :arrow_down: |
| [lib/promscrape/targets\_response.qtpl.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL3Byb21zY3JhcGUvdGFyZ2V0c19yZXNwb25zZS5xdHBsLmdv) | `0.00% <0.00%> (ø)` | |
| [lib/promscrape/targetstatus.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL3Byb21zY3JhcGUvdGFyZ2V0c3RhdHVzLmdv) | `12.26% <0.00%> (-0.81%)` | :arrow_down: |
| [lib/mergeset/block\_stream\_reader.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL21lcmdlc2V0L2Jsb2NrX3N0cmVhbV9yZWFkZXIuZ28=) | `49.30% <0.00%> (-13.20%)` | :arrow_down: |
| [lib/storage/part.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvcGFydC5nbw==) | `75.86% <0.00%> (-1.73%)` | :arrow_down: |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `64.75% <0.00%> (-0.94%)` | :arrow_down: |
| [lib/decimal/decimal.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL2RlY2ltYWwvZGVjaW1hbC5nbw==) | `93.35% <0.00%> (-0.79%)` | :arrow_down: |
| [lib/storage/index\_db.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvaW5kZXhfZGIuZ28=) | `64.07% <0.00%> (-0.06%)` | :arrow_down: |
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `87.98% <0.00%> (ø)` | |
| ... and [2 more](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961/diff?src=pr&el=tree-more) | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=footer). Last update [e7d8d84...eefa797](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/961?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/README.md b/README.md
index 045c3d000..cb81548c1 100644
--- a/README.md
+++ b/README.md
@@ -1332,7 +1332,8 @@ An alternative solution is to query `/internal/resetRollupResultCache` url after
 the query cache, which could contain incomplete data cached during the backfilling.
 
 Yet another solution is to increase `-search.cacheTimestampOffset` flag value in order to disable caching
-for data with timestamps close to the current time.
+for data with timestamps close to the current time. Single-node VictoriaMetrics automatically resets response
+cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to it.
 
 
 ## Data updates
diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go
index c74d74768..0aa5ec3f5 100644
--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -3,12 +3,15 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
+	"path"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -50,7 +53,7 @@ func main() {
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
-	vmstorage.Init()
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
 	startSelfScraper()
@@ -80,8 +83,16 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.RequestURI == "/" {
-		fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+	if r.URL.Path == "/" {
+		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics.</h2></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://victoriametrics.github.io/'>https://victoriametrics.github.io/</a></br>")
+		fmt.Fprintf(w, "usefull apis: </br>")
+		writeAPIHelp(w, [][]string{
+			{"/targets", "discovered targets list"},
+			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"/metrics", "available service metrics"},
+			{"/api/v1/status/tsdb", "tsdb status page"},
+		})
 		return true
 	}
 	if vminsert.RequestHandler(w, r) {
@@ -95,3 +106,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	return false
 }
+
+func writeAPIHelp(w io.Writer, pathList [][]string) {
+	pathPrefix := httpserver.GetPathPrefix()
+	for _, p := range pathList {
+		p, doc := p[0], p[1]
+		p = path.Join(pathPrefix, p)
+		fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
+	}
+}
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index a66bbce22..3c429c15c 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -212,9 +212,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
-		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		showOnlyUnhealthy, _ := strconv.ParseBool(r.FormValue("show_only_unhealthy"))
+		if accept := r.Header.Get("accept"); strings.Contains(accept, "text/html") {
+			promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels, showOnlyUnhealthy, "html")
+			return true
+		}
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels, showOnlyUnhealthy, "plain")
 		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index 3bfd7f2c3..c8f27da05 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -155,9 +155,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
-		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		showOnlyUnhealthy, _ := strconv.ParseBool(r.FormValue("show_only_unhealthy"))
+		if accept := r.Header.Get("accept"); strings.Contains(accept, "text/html") {
+			promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels, showOnlyUnhealthy, "html")
+			return true
+		}
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels, showOnlyUnhealthy, "plain")
 		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
diff --git a/app/vmselect/promql/rollup_result_cache.go b/app/vmselect/promql/rollup_result_cache.go
index db6f28931..8ef54398e 100644
--- a/app/vmselect/promql/rollup_result_cache.go
+++ b/app/vmselect/promql/rollup_result_cache.go
@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/VictoriaMetrics/metrics"
@@ -25,6 +26,39 @@ var (
 		"due to time synchronization issues between VictoriaMetrics and data sources")
 )
 
+// ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
+func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
+	checkRollupResultCacheResetOnce.Do(func() {
+		go checkRollupResultCacheReset()
+	})
+	minTimestamp := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds() + checkRollupResultCacheResetInterval.Milliseconds()
+	needCacheReset := false
+	for i := range mrs {
+		if mrs[i].Timestamp < minTimestamp {
+			needCacheReset = true
+			break
+		}
+	}
+	if needCacheReset {
+		// Do not call ResetRollupResultCache() here, since it may be heavy when frequently called.
+		atomic.StoreUint32(&needRollupResultCacheReset, 1)
+	}
+}
+
+func checkRollupResultCacheReset() {
+	for {
+		time.Sleep(checkRollupResultCacheResetInterval)
+		if atomic.SwapUint32(&needRollupResultCacheReset, 0) > 0 {
+			ResetRollupResultCache()
+		}
+	}
+}
+
+const checkRollupResultCacheResetInterval = 5 * time.Second
+
+var needRollupResultCacheReset uint32
+var checkRollupResultCacheResetOnce sync.Once
+
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024*1024, time.Hour), // This is a cache for testing.
 }
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index cee1db78c..920708422 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -57,19 +57,20 @@ func CheckTimeRange(tr storage.TimeRange) error {
 }
 
 // Init initializes vmstorage.
-func Init() {
-	InitWithoutMetrics()
+func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+	InitWithoutMetrics(resetCacheIfNeeded)
 	registerStorageMetrics()
 }
 
 // InitWithoutMetrics must be called instead of Init inside tests.
 //
 // This allows multiple Init / Stop cycles.
-func InitWithoutMetrics() {
+func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
 
+	resetResponseCacheIfNeeded = resetCacheIfNeeded
 	storage.SetFinalMergeDelay(*finalMergeDelay)
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
@@ -105,8 +106,12 @@ var Storage *storage.Storage
 // Use syncwg instead of sync, since Add is called from concurrent goroutines.
 var WG syncwg.WaitGroup
 
+// resetResponseCacheIfNeeded is a callback for automatic resetting of response cache if needed.
+var resetResponseCacheIfNeeded func(mrs []storage.MetricRow)
+
 // AddRows adds mrs to the storage.
 func AddRows(mrs []storage.MetricRow) error {
+	resetResponseCacheIfNeeded(mrs)
 	WG.Add(1)
 	err := Storage.AddRows(mrs, uint8(*precisionBits))
 	WG.Done()
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3c2afbff7..f8b88b481 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 # tip
 
+* FEATURE: automatically reset response cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to VictoriaMetrics. This makes unnecessary disabling response cache during data backfilling or resetting it after backfilling is complete as described [in these docs](https://victoriametrics.github.io/#backfilling). This feature applies only to single-node VictoriaMetrics. It doesn't apply to cluster version of VictoriaMetrics because `vminsert` nodes don't know about `vmselect` nodes where the response cache must be reset.
 * FEATURE: allow multiple whitespace chars between measurements, fields and timestamp when parsing InfluxDB line protocol.
   Though [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) denies multiple whitespace chars between these entities,
   some apps improperly put multiple whitespace chars. This workaround allows accepting data from such apps.
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 045c3d000..cb81548c1 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -1332,7 +1332,8 @@ An alternative solution is to query `/internal/resetRollupResultCache` url after
 the query cache, which could contain incomplete data cached during the backfilling.
 
 Yet another solution is to increase `-search.cacheTimestampOffset` flag value in order to disable caching
-for data with timestamps close to the current time.
+for data with timestamps close to the current time. Single-node VictoriaMetrics automatically resets response
+cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to it.
 
 
 ## Data updates
diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go
index daa8c8e59..6a760af5b 100644
--- a/lib/httpserver/httpserver.go
+++ b/lib/httpserver/httpserver.go
@@ -274,9 +274,12 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 }
 
 func getCanonicalPath(path string) (string, error) {
-	if len(*pathPrefix) == 0 {
+	if len(*pathPrefix) == 0 || path == "/" {
 		return path, nil
 	}
+	if *pathPrefix == path {
+		return "/", nil
+	}
 	prefix := *pathPrefix
 	if !strings.HasSuffix(prefix, "/") {
 		prefix = prefix + "/"
@@ -573,3 +576,8 @@ func isTrivialNetworkError(err error) bool {
 func IsTLS() bool {
 	return *tlsEnable
 }
+
+// GetPathPrefix - returns http server path prefix.
+func GetPathPrefix() string {
+	return *pathPrefix
+}
diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl
new file mode 100644
index 000000000..7f3d1efe3
--- /dev/null
+++ b/lib/promscrape/targets_response.qtpl
@@ -0,0 +1,104 @@
+{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+%}
+
+{% stripspace %}
+
+{% func TargetsResponsePlain (jts []jobTargetsStatuses,  showOriginLabels bool) -%}
+
+{% for _, js := range jts %}
+job={%q= js.job %}{% space %} ({%d js.upCount %}/{%d js.targetsTotal %} {% space %} up)
+{% newline %}
+{% for _, ts := range js.targetsStatus %}
+    {% code
+     labels :=  promLabelsString(ts.labels)
+     ol := promLabelsString(ts.originalLabels)
+    %}
+{%s= "\t" %}state={% if ts.up %}up{% else %}down{% endif %},
+    {% space %} endpoint={%s= ts.endpoint %},
+    {% space %} labels={%s= labels %}
+    {% if showOriginLabels %},{% space %} originalLabels={%s= ol %}{% endif %},
+    {% space %} last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago,
+    {% space %} scrape_duration={%f.3 float64(ts.scrapeDuration.Seconds()) %}s,
+    {% space %} error={%q= ts.error %}
+    {% newline %}
+{% endfor %}
+{% endfor %}
+{% newline %}
+
+{% endfunc %}
+
+{% func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) %}
+<!DOCTYPE html>
+<style>
+    .border{
+        border-collapse: collapse;
+        border: 1px solid black;
+    }
+    .table-row:hover{
+        background-color: #f5f5f5;
+    }
+</style>
+<html lang="en">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+        <title>VictoriaMetrics Database</title>
+    </head>
+    <body>
+    <h1>Targets</h1>
+    <div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons">
+      <label class="btn">
+        <input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}';" {% if !onlyUnhealthy %}checked {% endif %}> All
+      </label>
+      <label class="btn">
+        <input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}?show_only_unhealthy=true';" {% if onlyUnhealthy %}checked {% endif %}> Unhealthy
+      </label>
+      <br />
+  </div>
+  {% for _,js :=range jts %}
+  		    <div class="table-container">
+                <h2 class="job_header danger">
+                  <a id="job-{%q= js.job %}" >{%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)</a>
+                </h2>
+                <table class="table-bordered table-hover border">
+                  <thead class="job_details border">
+                    <tr class="table-row border">
+                      <th class="border">Endpoint</th>
+                      <th class="border">State</th>
+                      <th class="border">Labels</th>
+                      <th class="border">Last Scrape</th>
+                      <th class="border">Scrape Duration</th>
+                      <th class="border">Error</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                  {% for _, ts := range js.targetsStatus %}
+                    {% if onlyUnhealthy && ts.up %} {% continue %} {% endif %}
+                    <tr class="table-row border">
+                      <td class="endpoint border">
+                        <a href="{%s= ts.endpoint %}">{%s= ts.endpoint %}</a><br>
+                      </td>
+                      <td class="state border">
+                        <span class="state_indicator">{% if ts.up %}UP{% else %}DOWN{% endif %}</span>
+                      </td>
+                      <td class="labels border", title="Original {% space %} labels: {% space %} {%= formatLabel(ts.originalLabels) %}">
+                           {%= formatLabel(ts.labels) %}
+                      </td>
+                      <td class="last-scrape border">{%s ts.lastScrapeTime.String() %} ago</td>
+                      <td class="scrape-duration border">{%s ts.scrapeDuration.String() %}</td>
+                      <td class="errors border"><span class="alert alert-danger state_indicator">{%s= ts.error %}</span></td>
+                    </tr>
+                  {% endfor %}
+                  </tbody>
+                </table>
+              </div>
+  {% endfor %}
+    </body>
+</html>
+{% endfunc %}
+
+{% func formatLabel(labels []prompbmarshal.Label) %}
+{% for _, label := range labels %}
+ {% space %} {%s label.Name %}={%q label.Value %} {% space %}
+{% endfor %}
+{% endfunc %}
+{% endstripspace %}
\ No newline at end of file
diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go
new file mode 100644
index 000000000..f5e319f06
--- /dev/null
+++ b/lib/promscrape/targets_response.qtpl.go
@@ -0,0 +1,343 @@
+// Code generated by qtc from "targets_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line lib/promscrape/targets_response.qtpl:1
+package promscrape
+
+//line lib/promscrape/targets_response.qtpl:1
+import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+
+//line lib/promscrape/targets_response.qtpl:6
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line lib/promscrape/targets_response.qtpl:6
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line lib/promscrape/targets_response.qtpl:6
+func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) {
+//line lib/promscrape/targets_response.qtpl:8
+	for _, js := range jts {
+//line lib/promscrape/targets_response.qtpl:8
+		qw422016.N().S(`job=`)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().S(`(`)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().D(js.upCount)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().S(`/`)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().D(js.targetsTotal)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:9
+		qw422016.N().S(`up)`)
+//line lib/promscrape/targets_response.qtpl:10
+		qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:11
+		for _, ts := range js.targetsStatus {
+//line lib/promscrape/targets_response.qtpl:13
+			labels := promLabelsString(ts.labels)
+			ol := promLabelsString(ts.originalLabels)
+
+//line lib/promscrape/targets_response.qtpl:16
+			qw422016.N().S("\t")
+//line lib/promscrape/targets_response.qtpl:16
+			qw422016.N().S(`state=`)
+//line lib/promscrape/targets_response.qtpl:16
+			if ts.up {
+//line lib/promscrape/targets_response.qtpl:16
+				qw422016.N().S(`up`)
+//line lib/promscrape/targets_response.qtpl:16
+			} else {
+//line lib/promscrape/targets_response.qtpl:16
+				qw422016.N().S(`down`)
+//line lib/promscrape/targets_response.qtpl:16
+			}
+//line lib/promscrape/targets_response.qtpl:16
+			qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:17
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:17
+			qw422016.N().S(`endpoint=`)
+//line lib/promscrape/targets_response.qtpl:17
+			qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:17
+			qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:18
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:18
+			qw422016.N().S(`labels=`)
+//line lib/promscrape/targets_response.qtpl:18
+			qw422016.N().S(labels)
+//line lib/promscrape/targets_response.qtpl:19
+			if showOriginLabels {
+//line lib/promscrape/targets_response.qtpl:19
+				qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:19
+				qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:19
+				qw422016.N().S(`originalLabels=`)
+//line lib/promscrape/targets_response.qtpl:19
+				qw422016.N().S(ol)
+//line lib/promscrape/targets_response.qtpl:19
+			}
+//line lib/promscrape/targets_response.qtpl:19
+			qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:20
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:20
+			qw422016.N().S(`last_scrape=`)
+//line lib/promscrape/targets_response.qtpl:20
+			qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
+//line lib/promscrape/targets_response.qtpl:20
+			qw422016.N().S(`s ago,`)
+//line lib/promscrape/targets_response.qtpl:21
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:21
+			qw422016.N().S(`scrape_duration=`)
+//line lib/promscrape/targets_response.qtpl:21
+			qw422016.N().FPrec(float64(ts.scrapeDuration.Seconds()), 3)
+//line lib/promscrape/targets_response.qtpl:21
+			qw422016.N().S(`s,`)
+//line lib/promscrape/targets_response.qtpl:22
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:22
+			qw422016.N().S(`error=`)
+//line lib/promscrape/targets_response.qtpl:22
+			qw422016.N().Q(ts.error)
+//line lib/promscrape/targets_response.qtpl:23
+			qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:24
+		}
+//line lib/promscrape/targets_response.qtpl:25
+	}
+//line lib/promscrape/targets_response.qtpl:26
+	qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:28
+func WriteTargetsResponsePlain(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) {
+//line lib/promscrape/targets_response.qtpl:28
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:28
+	StreamTargetsResponsePlain(qw422016, jts, showOriginLabels)
+//line lib/promscrape/targets_response.qtpl:28
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:28
+func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) string {
+//line lib/promscrape/targets_response.qtpl:28
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:28
+	WriteTargetsResponsePlain(qb422016, jts, showOriginLabels)
+//line lib/promscrape/targets_response.qtpl:28
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:28
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:28
+	return qs422016
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:30
+func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
+//line lib/promscrape/targets_response.qtpl:30
+	qw422016.N().S(`<!DOCTYPE html><style>.border{border-collapse: collapse;border: 1px solid black;}.table-row:hover{background-color: #f5f5f5;}</style><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>VictoriaMetrics Database</title></head><body><h1>Targets</h1><div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons"><label class="btn"><input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:50
+	qw422016.N().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:50
+	qw422016.N().S(`';"`)
+//line lib/promscrape/targets_response.qtpl:50
+	if !onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:50
+		qw422016.N().S(`checked`)
+//line lib/promscrape/targets_response.qtpl:50
+	}
+//line lib/promscrape/targets_response.qtpl:50
+	qw422016.N().S(`> All</label><label class="btn"><input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:53
+	qw422016.N().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:53
+	qw422016.N().S(`?show_only_unhealthy=true';"`)
+//line lib/promscrape/targets_response.qtpl:53
+	if onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:53
+		qw422016.N().S(`checked`)
+//line lib/promscrape/targets_response.qtpl:53
+	}
+//line lib/promscrape/targets_response.qtpl:53
+	qw422016.N().S(`> Unhealthy</label><br /></div>`)
+//line lib/promscrape/targets_response.qtpl:57
+	for _, js := range jts {
+//line lib/promscrape/targets_response.qtpl:57
+		qw422016.N().S(`<div class="table-container"><h2 class="job_header danger"><a id="job-`)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().S(`" >`)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().S(`(`)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().D(js.upCount)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().S(`/`)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().D(js.targetsTotal)
+//line lib/promscrape/targets_response.qtpl:60
+		qw422016.N().S(`up)</a></h2><table class="table-bordered table-hover border"><thead class="job_details border"><tr class="table-row border"><th class="border">Endpoint</th><th class="border">State</th><th class="border">Labels</th><th class="border">Last Scrape</th><th class="border">Scrape Duration</th><th class="border">Error</th></tr></thead><tbody>`)
+//line lib/promscrape/targets_response.qtpl:74
+		for _, ts := range js.targetsStatus {
+//line lib/promscrape/targets_response.qtpl:75
+			if onlyUnhealthy && ts.up {
+//line lib/promscrape/targets_response.qtpl:75
+				continue
+//line lib/promscrape/targets_response.qtpl:75
+			}
+//line lib/promscrape/targets_response.qtpl:75
+			qw422016.N().S(`<tr class="table-row border"><td class="endpoint border"><a href="`)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().S(`">`)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().S(`</a><br></td><td class="state border"><span class="state_indicator">`)
+//line lib/promscrape/targets_response.qtpl:81
+			if ts.up {
+//line lib/promscrape/targets_response.qtpl:81
+				qw422016.N().S(`UP`)
+//line lib/promscrape/targets_response.qtpl:81
+			} else {
+//line lib/promscrape/targets_response.qtpl:81
+				qw422016.N().S(`DOWN`)
+//line lib/promscrape/targets_response.qtpl:81
+			}
+//line lib/promscrape/targets_response.qtpl:81
+			qw422016.N().S(`</span></td><td class="labels border", title="Original`)
+//line lib/promscrape/targets_response.qtpl:83
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:83
+			qw422016.N().S(`labels:`)
+//line lib/promscrape/targets_response.qtpl:83
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:83
+			streamformatLabel(qw422016, ts.originalLabels)
+//line lib/promscrape/targets_response.qtpl:83
+			qw422016.N().S(`">`)
+//line lib/promscrape/targets_response.qtpl:84
+			streamformatLabel(qw422016, ts.labels)
+//line lib/promscrape/targets_response.qtpl:84
+			qw422016.N().S(`</td><td class="last-scrape border">`)
+//line lib/promscrape/targets_response.qtpl:86
+			qw422016.E().S(ts.lastScrapeTime.String())
+//line lib/promscrape/targets_response.qtpl:86
+			qw422016.N().S(`ago</td><td class="scrape-duration border">`)
+//line lib/promscrape/targets_response.qtpl:87
+			qw422016.E().S(ts.scrapeDuration.String())
+//line lib/promscrape/targets_response.qtpl:87
+			qw422016.N().S(`</td><td class="errors border"><span class="alert alert-danger state_indicator">`)
+//line lib/promscrape/targets_response.qtpl:88
+			qw422016.N().S(ts.error)
+//line lib/promscrape/targets_response.qtpl:88
+			qw422016.N().S(`</span></td></tr>`)
+//line lib/promscrape/targets_response.qtpl:90
+		}
+//line lib/promscrape/targets_response.qtpl:90
+		qw422016.N().S(`</tbody></table></div>`)
+//line lib/promscrape/targets_response.qtpl:94
+	}
+//line lib/promscrape/targets_response.qtpl:94
+	qw422016.N().S(`</body></html>`)
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:97
+func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
+//line lib/promscrape/targets_response.qtpl:97
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:97
+	StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy)
+//line lib/promscrape/targets_response.qtpl:97
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:97
+func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string {
+//line lib/promscrape/targets_response.qtpl:97
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:97
+	WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy)
+//line lib/promscrape/targets_response.qtpl:97
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:97
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:97
+	return qs422016
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:99
+func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
+//line lib/promscrape/targets_response.qtpl:100
+	for _, label := range labels {
+//line lib/promscrape/targets_response.qtpl:101
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:101
+		qw422016.E().S(label.Name)
+//line lib/promscrape/targets_response.qtpl:101
+		qw422016.N().S(`=`)
+//line lib/promscrape/targets_response.qtpl:101
+		qw422016.E().Q(label.Value)
+//line lib/promscrape/targets_response.qtpl:101
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:102
+	}
+//line lib/promscrape/targets_response.qtpl:103
+}
+
+//line lib/promscrape/targets_response.qtpl:103
+func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) {
+//line lib/promscrape/targets_response.qtpl:103
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:103
+	streamformatLabel(qw422016, labels)
+//line lib/promscrape/targets_response.qtpl:103
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:103
+}
+
+//line lib/promscrape/targets_response.qtpl:103
+func formatLabel(labels []prompbmarshal.Label) string {
+//line lib/promscrape/targets_response.qtpl:103
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:103
+	writeformatLabel(qb422016, labels)
+//line lib/promscrape/targets_response.qtpl:103
+	qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:103
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:103
+	return qs422016
+//line lib/promscrape/targets_response.qtpl:103
+}
diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go
index 0a75f00e4..cd10293cf 100644
--- a/lib/promscrape/targetstatus.go
+++ b/lib/promscrape/targetstatus.go
@@ -4,11 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
+	"path"
 	"sort"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -19,9 +22,16 @@ var maxDroppedTargets = flag.Int("promscrape.maxDroppedTargets", 1000, "The maxi
 
 var tsmGlobal = newTargetStatusMap()
 
-// WriteHumanReadableTargetsStatus writes human-readable status for all the scrape targets to w.
-func WriteHumanReadableTargetsStatus(w io.Writer, showOriginalLabels bool) {
-	tsmGlobal.WriteHumanReadable(w, showOriginalLabels)
+// WriteHumanReadableTargetsStatus writes human-readable status for all the scrape targets to w with given format and options.
+func WriteHumanReadableTargetsStatus(w http.ResponseWriter, showOriginalLabels, showOnlyUnhealthy bool, format string) {
+	switch format {
+	case "plain":
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		tsmGlobal.WriteTargetsPlain(w, showOriginalLabels)
+	case "html":
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		tsmGlobal.WriteTargetsHTML(w, showOnlyUnhealthy)
+	}
 }
 
 // WriteAPIV1Targets writes /api/v1/targets to w according to https://prometheus.io/docs/prometheus/latest/querying/api/#targets
@@ -162,64 +172,6 @@ func writeLabelsJSON(w io.Writer, labels []prompbmarshal.Label) {
 	fmt.Fprintf(w, `}`)
 }
 
-func (tsm *targetStatusMap) WriteHumanReadable(w io.Writer, showOriginalLabels bool) {
-	byJob := make(map[string][]targetStatus)
-	tsm.mu.Lock()
-	for _, st := range tsm.m {
-		job := st.sw.Job()
-		byJob[job] = append(byJob[job], *st)
-	}
-	tsm.mu.Unlock()
-
-	var jss []jobStatus
-	for job, statuses := range byJob {
-		jss = append(jss, jobStatus{
-			job:      job,
-			statuses: statuses,
-		})
-	}
-	sort.Slice(jss, func(i, j int) bool {
-		return jss[i].job < jss[j].job
-	})
-
-	for _, js := range jss {
-		sts := js.statuses
-		sort.Slice(sts, func(i, j int) bool {
-			return sts[i].sw.ScrapeURL < sts[j].sw.ScrapeURL
-		})
-		ups := 0
-		for _, st := range sts {
-			if st.up {
-				ups++
-			}
-		}
-		fmt.Fprintf(w, "job=%q (%d/%d up)\n", js.job, ups, len(sts))
-		for _, st := range sts {
-			state := "up"
-			if !st.up {
-				state = "down"
-			}
-			labelsStr := st.sw.LabelsString()
-			if showOriginalLabels {
-				labelsStr += ", originalLabels=" + promLabelsString(st.sw.OriginalLabels)
-			}
-			lastScrape := st.getDurationFromLastScrape()
-			errMsg := ""
-			if st.err != nil {
-				errMsg = st.err.Error()
-			}
-			fmt.Fprintf(w, "\tstate=%s, endpoint=%s, labels=%s, last_scrape=%.3fs ago, scrape_duration=%.3fs, error=%q\n",
-				state, st.sw.ScrapeURL, labelsStr, lastScrape.Seconds(), float64(st.scrapeDuration)/1000, errMsg)
-		}
-	}
-	fmt.Fprintf(w, "\n")
-}
-
-type jobStatus struct {
-	job      string
-	statuses []targetStatus
-}
-
 type targetStatus struct {
 	sw             ScrapeWork
 	up             bool
@@ -303,3 +255,84 @@ func (dt *droppedTargets) WriteDroppedTargetsJSON(w io.Writer) {
 var droppedTargetsMap = &droppedTargets{
 	m: make(map[string]droppedTarget),
 }
+
+type jobTargetStatus struct {
+	up             bool
+	endpoint       string
+	labels         []prompbmarshal.Label
+	originalLabels []prompbmarshal.Label
+	lastScrapeTime time.Duration
+	scrapeDuration time.Duration
+	error          string
+}
+
+type jobTargetsStatuses struct {
+	job           string
+	upCount       int
+	targetsTotal  int
+	targetsStatus []jobTargetStatus
+}
+
+func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses {
+	byJob := make(map[string][]targetStatus)
+	tsm.mu.Lock()
+	for _, st := range tsm.m {
+		job := st.sw.Job()
+		byJob[job] = append(byJob[job], *st)
+	}
+	tsm.mu.Unlock()
+
+	var jts []jobTargetsStatuses
+	for job, statuses := range byJob {
+		sort.Slice(statuses, func(i, j int) bool {
+			return statuses[i].sw.ScrapeURL < statuses[j].sw.ScrapeURL
+		})
+		ups := 0
+		var targetsStatuses []jobTargetStatus
+		for _, ts := range statuses {
+			if ts.up {
+				ups++
+			}
+		}
+		for _, st := range statuses {
+			errMsg := ""
+			if st.err != nil {
+				errMsg = st.err.Error()
+			}
+			targetsStatuses = append(targetsStatuses, jobTargetStatus{
+				up:             st.up,
+				endpoint:       st.sw.ScrapeURL,
+				labels:         promrelabel.FinalizeLabels(nil, st.sw.Labels),
+				originalLabels: st.sw.OriginalLabels,
+				lastScrapeTime: st.getDurationFromLastScrape(),
+				scrapeDuration: time.Duration(st.scrapeDuration),
+				error:          errMsg,
+			})
+		}
+		jts = append(jts, jobTargetsStatuses{
+			job:           job,
+			upCount:       ups,
+			targetsTotal:  len(statuses),
+			targetsStatus: targetsStatuses,
+		})
+	}
+	sort.Slice(jts, func(i, j int) bool {
+		return jts[i].job < jts[j].job
+	})
+	return jts
+}
+
+// WriteTargetsHTML writes targets status grouped by job into writer w in html table,
+// accepts filter to show only unhealthy targets.
+func (tsm *targetStatusMap) WriteTargetsHTML(w io.Writer, showOnlyUnhealthy bool) {
+	jss := tsm.getTargetsStatusByJob()
+	targetsPath := path.Join(httpserver.GetPathPrefix(), "/targets")
+	WriteTargetsResponseHTML(w, jss, targetsPath, showOnlyUnhealthy)
+}
+
+// WriteTargetsPlain writes targets grouped by job into writer w in plain text,
+// accept filter to show original labels.
+func (tsm *targetStatusMap) WriteTargetsPlain(w io.Writer, showOriginalLabels bool) {
+	jss := tsm.getTargetsStatusByJob()
+	WriteTargetsResponsePlain(w, jss, showOriginalLabels)
+}


Test patch:
diff --git a/app/victoria-metrics/main_test.go b/app/victoria-metrics/main_test.go
index 6eb2c13c9..926157623 100644
--- a/app/victoria-metrics/main_test.go
+++ b/app/victoria-metrics/main_test.go
@@ -20,6 +20,7 @@ import (
 	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -129,7 +130,7 @@ func setUp() {
 	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
 	processFlags()
 	logger.Init()
-	vmstorage.InitWithoutMetrics()
+	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
 	go httpserver.Serve(*httpListenAddr, requestHandler)
@@ -192,7 +193,7 @@ func TestWriteRead(t *testing.T) {
 	time.Sleep(1 * time.Second)
 	vmstorage.Stop()
 	// open storage after stop in write
-	vmstorage.InitWithoutMetrics()
+	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
 	t.Run("read", testRead)
 }