Repository: VictoriaMetrics/VictoriaMetrics
Pull request #500 (2020-05-17 16:22:57)

dashboards: updates and fixes for cluster version
* The new update introduces new row "Troubleshooting" that
contains panels for churn rate and slow-queries/inserts/loads metrics. This row supposed to be reveal the cause of low performance or other issues;
* CPU panel got `short` units instead of `seconds`;
* Overview row was updated with panel showing bytes-per-datapoint stat;
* Overview row was updated with panel showing free disk space.

#494 #407 #397


Patch:
diff --git a/app/vmalert/Makefile b/app/vmalert/Makefile
index d49984ddf..0d9439398 100644
--- a/app/vmalert/Makefile
+++ b/app/vmalert/Makefile
@@ -52,7 +52,9 @@ publish-vmalert:
 	APP_NAME=vmalert $(MAKE) publish-via-docker
 
 test-vmalert:
-	go test -race -cover ./app/vmalert
+	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/datasource
+	go test -v -race -cover ./app/vmalert/notifier
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/testdata/rules0-good.rules \
diff --git a/app/vmalert/config.go b/app/vmalert/config.go
index e46c0c497..b84e7044d 100644
--- a/app/vmalert/config.go
+++ b/app/vmalert/config.go
@@ -32,8 +32,9 @@ func Parse(pathPatterns []string, validateAnnotations bool) ([]Group, error) {
 				return nil, fmt.Errorf("one file can not contain groups with the same name %s, filepath:%s", g.Name, file)
 			}
 			g.File = file
-			g.done = make(chan struct{})
-			g.finished = make(chan struct{})
+			g.doneCh = make(chan struct{})
+			g.finishedCh = make(chan struct{})
+			g.updateCh = make(chan Group)
 
 			groupsNames[g.Name] = struct{}{}
 			for _, rule := range g.Rules {
diff --git a/app/vmalert/group.go b/app/vmalert/group.go
index d4713496f..4d5e5a318 100644
--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -19,13 +19,16 @@ type Group struct {
 	File  string
 	Rules []*Rule
 
-	done     chan struct{}
-	finished chan struct{}
+	doneCh     chan struct{}
+	finishedCh chan struct{}
+	// channel accepts new Group obj
+	// which supposed to update current group
+	updateCh chan Group
 }
 
 // ID return unique group ID that consists of
 // rules file and group name
-func (g Group) ID() uint64 {
+func (g *Group) ID() uint64 {
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -47,8 +50,8 @@ func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time
 }
 
 // updateWith updates existing group with
-// passed group object. Must be called
-// under mutex lock.
+// passed group object.
+// Not thread-safe.
 func (g *Group) updateWith(newGroup Group) {
 	rulesRegistry := make(map[string]*Rule)
 	for _, nr := range newGroup.Rules {
@@ -66,7 +69,7 @@ func (g *Group) updateWith(newGroup Group) {
 
 		// copy all significant fields.
 		// alerts state isn't copied since
-		// it should be updated in next 2 Evals
+		// it should be updated in next 2 Execs
 		or.For = nr.For
 		or.Expr = nr.Expr
 		or.Labels = nr.Labels
@@ -106,11 +109,11 @@ var (
 )
 
 func (g *Group) close() {
-	if g.done == nil {
+	if g.doneCh == nil {
 		return
 	}
-	close(g.done)
-	<-g.finished
+	close(g.doneCh)
+	<-g.finishedCh
 }
 
 func (g *Group) start(ctx context.Context, interval time.Duration,
@@ -122,12 +125,14 @@ func (g *Group) start(ctx context.Context, interval time.Duration,
 		select {
 		case <-ctx.Done():
 			logger.Infof("group %q: context cancelled", g.Name)
-			close(g.finished)
+			close(g.finishedCh)
 			return
-		case <-g.done:
+		case <-g.doneCh:
 			logger.Infof("group %q: received stop signal", g.Name)
-			close(g.finished)
+			close(g.finishedCh)
 			return
+		case ng := <-g.updateCh:
+			g.updateWith(ng)
 		case <-t.C:
 			iterationTotal.Inc()
 			iterationStart := time.Now()
@@ -146,30 +151,47 @@ func (g *Group) start(ctx context.Context, interval time.Duration,
 
 				var alertsToSend []notifier.Alert
 				for _, a := range rule.alerts {
-					if a.State != notifier.StatePending {
+					switch a.State {
+					case notifier.StateFiring:
+						// set End to execStart + 3 intervals
+						// so notifier can resolve it automatically if `vmalert`
+						// won't be able to send resolve for some reason
+						a.End = execStart.Add(3 * interval)
+						alertsToSend = append(alertsToSend, *a)
+						pushToRW(rw, rule, a, execStart)
+					case notifier.StatePending:
+						pushToRW(rw, rule, a, execStart)
+					case notifier.StateInactive:
+						// set End to execStart to notify
+						// that it was just resolved
+						a.End = execStart
 						alertsToSend = append(alertsToSend, *a)
-					}
-					if a.State == notifier.StateInactive || rw == nil {
-						continue
-					}
-					tss := rule.AlertToTimeSeries(a, execStart)
-					for _, ts := range tss {
-						remoteWriteSent.Inc()
-						if err := rw.Push(ts); err != nil {
-							remoteWriteErrors.Inc()
-							logger.Errorf("failed to push timeseries to remotewrite: %s", err)
-						}
 					}
 				}
-				if len(alertsToSend) > 0 {
-					alertsSent.Add(len(alertsToSend))
-					if err := nr.Send(ctx, alertsToSend); err != nil {
-						alertsSendErrors.Inc()
-						logger.Errorf("failed to send alert for rule %q.%q: %s", g.Name, rule.Name, err)
-					}
+				if len(alertsToSend) == 0 {
+					continue
+				}
+				alertsSent.Add(len(alertsToSend))
+				if err := nr.Send(ctx, alertsToSend); err != nil {
+					alertsSendErrors.Inc()
+					logger.Errorf("failed to send alert for rule %q.%q: %s", g.Name, rule.Name, err)
 				}
 			}
 			iterationDuration.UpdateDuration(iterationStart)
 		}
 	}
 }
+
+func pushToRW(rw *remotewrite.Client, rule *Rule, a *notifier.Alert, timestamp time.Time) {
+	if rw == nil {
+		return
+	}
+	tss := rule.AlertToTimeSeries(a, timestamp)
+	remoteWriteSent.Add(len(tss))
+	for _, ts := range tss {
+		if err := rw.Push(ts); err != nil {
+			remoteWriteErrors.Inc()
+			logger.Errorf("failed to push timeseries to remotewrite: %s", err)
+		}
+	}
+}
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 9d71e8511..0452f4c07 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -121,7 +121,7 @@ func main() {
 	}()
 
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, (rh).handler)
+	go httpserver.Serve(*httpListenAddr, rh.handler)
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go
index 38504fd52..4429ca8d3 100644
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -87,8 +87,7 @@ func (m *manager) update(ctx context.Context, path []string, validate, restore b
 
 	m.groupsMu.Lock()
 	for _, og := range m.groups {
-		id := og.ID()
-		ng, ok := groupsRegistry[id]
+		ng, ok := groupsRegistry[og.ID()]
 		if !ok {
 			// old group is not present in new list
 			// and must be stopped and deleted
@@ -97,7 +96,7 @@ func (m *manager) update(ctx context.Context, path []string, validate, restore b
 			og = nil
 			continue
 		}
-		og.updateWith(ng)
+		og.updateCh <- ng
 		delete(groupsRegistry, ng.ID())
 	}
 
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index b883aa193..b1cd05018 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -18,6 +18,7 @@ type Alert struct {
 	Annotations map[string]string
 	State       AlertState
 
+	Expr  string
 	Start time.Time
 	End   time.Time
 	Value float64
@@ -52,14 +53,15 @@ func (as AlertState) String() string {
 type alertTplData struct {
 	Labels map[string]string
 	Value  float64
+	Expr   string
 }
 
-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}`
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
 
 // ExecTemplate executes the Alert template for give
 // map of annotations.
 func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
-	tplData := alertTplData{Value: a.Value, Labels: a.Labels}
+	tplData := alertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
 	return templateAnnotations(annotations, tplHeader, tplData)
 }
 
diff --git a/app/vmalert/notifier/template_func.go b/app/vmalert/notifier/template_func.go
index 45f5e349f..88547d1a5 100644
--- a/app/vmalert/notifier/template_func.go
+++ b/app/vmalert/notifier/template_func.go
@@ -142,6 +142,12 @@ func InitTemplateFunc(externalURL *url.URL) {
 		"externalURL": func() string {
 			return externalURL.String()
 		},
+		"pathEscape": func(u string) string {
+			return url.PathEscape(u)
+		},
+		"queryEscape": func(q string) string {
+			return url.QueryEscape(q)
+		},
 	}
 }
 
diff --git a/app/vmalert/rule.go b/app/vmalert/rule.go
index 08d1d64d1..c604361cd 100644
--- a/app/vmalert/rule.go
+++ b/app/vmalert/rule.go
@@ -71,7 +71,7 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
 	}
 
 	for h, a := range r.alerts {
-		// cleanup inactive alerts from previous Eval
+		// cleanup inactive alerts from previous Exec
 		if a.State == notifier.StateInactive {
 			delete(r.alerts, h)
 		}
@@ -109,19 +109,19 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
 		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(r.alerts, h)
+				continue
+			}
 			a.State = notifier.StateInactive
-			// set endTime to last execution time
-			// so it can be sent by notifier on next step
-			a.End = r.lastExecTime
 			continue
 		}
 		if a.State == notifier.StatePending && time.Since(a.Start) >= r.For {
 			a.State = notifier.StateFiring
 			alertsFired.Inc()
 		}
-		if a.State == notifier.StateFiring {
-			a.End = r.lastExecTime.Add(3 * *evaluationInterval)
-		}
 	}
 	return nil
 }
@@ -152,6 +152,7 @@ func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
 		Labels:  map[string]string{},
 		Value:   m.Value,
 		Start:   time.Now(),
+		Expr:    r.Expr,
 		// TODO: support End time
 	}
 	for _, l := range m.Labels {
@@ -293,7 +294,7 @@ func newTimeSeries(value float64, labels map[string]string, timestamp time.Time)
 
 // Restore restores the state of active alerts basing on previously written timeseries.
 // Restore restores only Start field. Field State will be always Pending and supposed
-// to be updated on next Eval, as well as Value field.
+// to be updated on next Exec, as well as Value field.
 func (r *Rule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
 	if q == nil {
 		return fmt.Errorf("querier is nil")
@@ -313,7 +314,7 @@ func (r *Rule) Restore(ctx context.Context, q datasource.Querier, lookback time.
 		labels := m.Labels
 		m.Labels = make([]datasource.Label, 0)
 		// drop all extra labels, so hash key will
-		// be identical to timeseries received in Eval
+		// be identical to timeseries received in Exec
 		for _, l := range labels {
 			if l.Name == alertNameLabel {
 				continue
diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json
index 79b800d1c..7c2428c80 100644
--- a/dashboards/victoriametrics.json
+++ b/dashboards/victoriametrics.json
@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "6.7.1"
+      "version": "6.7.2"
     },
     {
       "type": "panel",
@@ -54,12 +54,12 @@
       }
     ]
   },
-  "description": "Overview for cluster VictoriaMetrics v1.34.0 or higher",
+  "description": "Overview for cluster VictoriaMetrics v1.35.5 or higher",
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1585342925438,
+  "iteration": 1589730921459,
   "links": [
     {
       "icon": "doc",
@@ -122,7 +122,7 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 8,
+        "w": 6,
         "x": 0,
         "y": 1
       },
@@ -206,8 +206,8 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 9,
-        "x": 8,
+        "w": 6,
+        "x": 6,
         "y": 1
       },
       "id": 35,
@@ -290,8 +290,8 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 7,
-        "x": 17,
+        "w": 6,
+        "x": 12,
         "y": 1
       },
       "id": 32,
@@ -353,6 +353,93 @@
       ],
       "valueName": "current"
     },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "$ds",
+      "decimals": 2,
+      "description": "Average disk usage per datapoint.",
+      "format": "bytes",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 6,
+        "x": 18,
+        "y": 1
+      },
+      "id": 112,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": true,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "sum(vm_rows{job=\"$job_storage\", type!=\"indexdb\"}) / sum(vm_data_size_bytes{job=\"$job_storage\", type!=\"indexdb\"})",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Bytes per point",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
     {
       "columns": [],
       "datasource": "$ds",
@@ -730,7 +817,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "Shows how many of new time-series are created every second.  High churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
+      "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -740,19 +827,23 @@
         "y": 18
       },
       "hiddenSeries": false,
-      "id": 102,
+      "id": 52,
       "legend": {
-        "avg": false,
-        "current": false,
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
         "max": false,
         "min": false,
         "show": true,
+        "sort": "current",
+        "sortDesc": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
-      "nullPointMode": "null",
+      "links": [],
+      "nullPointMode": "null as zero",
       "options": {
         "dataLinks": []
       },
@@ -766,8 +857,10 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[5m]))",
-          "legendFormat": "churn rate",
+          "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance.*\"}[5m])) by (path) > 0",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{path}}",
           "refId": "A"
         }
       ],
@@ -775,10 +868,10 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Churn rate ($instance)",
+      "title": "Requests error rate ($instance)",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -912,7 +1005,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+      "description": "RPC errors are interconnection errors between cluster components. Errors rate should be 0 if network connection is stable and all components are up and operational.",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -922,7 +1015,7 @@
         "y": 26
       },
       "hiddenSeries": false,
-      "id": 52,
+      "id": 44,
       "legend": {
         "alignAsTable": true,
         "avg": true,
@@ -930,20 +1023,18 @@
         "max": false,
         "min": false,
         "show": true,
-        "sort": "current",
-        "sortDesc": true,
         "total": false,
         "values": true
       },
       "lines": true,
       "linewidth": 1,
       "links": [],
-      "nullPointMode": "null as zero",
+      "nullPointMode": "null",
       "options": {
         "dataLinks": []
       },
       "percentage": false,
-      "pointradius": 2,
+      "pointradius": 1,
       "points": false,
       "renderer": "flot",
       "seriesOverrides": [],
@@ -952,21 +1043,42 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance.*\"}[5m])) by (path) > 0",
+          "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
           "format": "time_series",
           "intervalFactor": 1,
-          "legendFormat": "{{path}}",
+          "legendFormat": "Connection",
           "refId": "A"
+        },
+        {
+          "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Dial",
+          "refId": "B"
+        },
+        {
+          "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Handshake",
+          "refId": "E"
+        },
+        {
+          "expr": "sum(rate(vm_rpc_reroute_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Reroute",
+          "refId": "C"
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Requests error rate ($instance)",
+      "title": "RPC errors ($instance)",
       "tooltip": {
         "shared": true,
-        "sort": 2,
+        "sort": 0,
         "value_type": "individual"
       },
       "type": "graph",
@@ -979,7 +1091,7 @@
       },
       "yaxes": [
         {
-          "format": "short",
+          "format": "rps",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1103,7 +1215,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "RPC errors are interconnection errors between cluster components. Errors rate should be 0 if network connection is stable and all components are up and operational.",
+      "description": "Shows amount of remaining disk space at `-storageDataPath`",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -1113,16 +1225,15 @@
         "y": 34
       },
       "hiddenSeries": false,
-      "id": 44,
+      "id": 110,
       "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": true,
+        "avg": false,
+        "current": false,
         "max": false,
         "min": false,
         "show": true,
         "total": false,
-        "values": true
+        "values": false
       },
       "lines": true,
       "linewidth": 1,
@@ -1132,48 +1243,28 @@
         "dataLinks": []
       },
       "percentage": false,
-      "pointradius": 1,
+      "pointradius": 2,
       "points": false,
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": false,
+      "stack": true,
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
+          "expr": "sum(vm_free_disk_space_bytes{job=\"$job_storage\", instance=~\"$instance\"}) by(instance)",
           "format": "time_series",
+          "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "Connection",
+          "legendFormat": "{{instance}}",
           "refId": "A"
-        },
-        {
-          "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Dial",
-          "refId": "B"
-        },
-        {
-          "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Handshake",
-          "refId": "E"
-        },
-        {
-          "expr": "sum(rate(vm_rpc_reroute_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[5m]))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Reroute",
-          "refId": "C"
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "RPC errors ($instance)",
+      "title": "Free disk space ($instance)",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1189,11 +1280,11 @@
       },
       "yaxes": [
         {
-          "format": "rps",
+          "format": "bytes",
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -1289,31 +1380,301 @@
           "min": null,
           "show": true
         },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 42
-      },
-      "id": 46,
-      "panels": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 42
+      },
+      "id": 46,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 3
+          },
+          "hiddenSeries": false,
+          "id": 64,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(instance)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "CPU ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "Amount of used memory (resident)",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 3
+          },
+          "hiddenSeries": false,
+          "id": 66,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}) by (instance)",
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Memory usage ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "bytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 11
+          },
+          "hiddenSeries": false,
+          "id": 68,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Goroutines ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": 0,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -1325,17 +1686,17 @@
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 43
+            "x": 12,
+            "y": 11
           },
           "hiddenSeries": false,
-          "id": 64,
+          "id": 70,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -1356,9 +1717,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(instance)",
+              "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
               "format": "time_series",
-              "intervalFactor": 1,
+              "intervalFactor": 2,
               "legendFormat": "{{instance}}",
               "refId": "A"
             }
@@ -1367,7 +1728,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "CPU ($instance)",
+          "title": "Threads ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1383,11 +1744,12 @@
           },
           "yaxes": [
             {
-              "format": "s",
+              "decimals": 0,
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -1410,17 +1772,16 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "Amount of used memory (resident)",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 43
+            "x": 0,
+            "y": 19
           },
           "hiddenSeries": false,
-          "id": 66,
+          "id": 72,
           "legend": {
             "avg": false,
             "current": false,
@@ -1447,7 +1808,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}) by (instance)",
+              "expr": "max(go_gc_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"1\"}) by(instance)",
+              "format": "time_series",
+              "intervalFactor": 2,
               "legendFormat": "{{instance}}",
               "refId": "A"
             }
@@ -1456,7 +1819,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Memory usage ($instance)",
+          "title": "GC duration ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1472,11 +1835,11 @@
           },
           "yaxes": [
             {
-              "format": "bytes",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -1492,35 +1855,50 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "Resource usage ($job)",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 43
+      },
+      "id": 106,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
+          "description": "Shows how many of new time-series are created every second.  High churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 3
           },
           "hiddenSeries": false,
-          "id": 68,
+          "id": 102,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": false,
+            "show": true,
             "total": false,
             "values": false
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
             "dataLinks": []
@@ -1535,10 +1913,8 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{instance}}",
+              "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[5m]))",
+              "legendFormat": "churn rate",
               "refId": "A"
             }
           ],
@@ -1546,7 +1922,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Goroutines ($instance)",
+          "title": "Churn rate ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1562,12 +1938,11 @@
           },
           "yaxes": [
             {
-              "decimals": 0,
               "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -1590,16 +1965,17 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
+          "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 3
           },
           "hiddenSeries": false,
-          "id": 70,
+          "id": 107,
           "legend": {
             "avg": false,
             "current": false,
@@ -1611,7 +1987,6 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
             "dataLinks": []
@@ -1626,10 +2001,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{instance}}",
+              "expr": "sum(rate(vm_slow_queries_total{job=\"$job_select\", instance=~\"$instance\"}[5m]))",
+              "interval": "",
+              "legendFormat": "slow queries rate",
               "refId": "A"
             }
           ],
@@ -1637,7 +2011,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Threads ($instance)",
+          "title": "Slow queries ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1653,12 +2027,11 @@
           },
           "yaxes": [
             {
-              "decimals": 0,
               "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -1681,16 +2054,17 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
+          "description": "The number of slow inserts during the last 5 minutes. If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series.\n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 11
           },
           "hiddenSeries": false,
-          "id": 72,
+          "id": 108,
           "legend": {
             "avg": false,
             "current": false,
@@ -1702,7 +2076,6 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
             "dataLinks": []
@@ -1717,10 +2090,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(go_gc_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"1\"}) by(instance)",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{instance}}",
+              "expr": "sum(increase(vm_slow_row_inserts_total{job=\"$job_storage\", instance=~\"$instance\"}[5m]))",
+              "interval": "",
+              "legendFormat": "slow inserts",
               "refId": "A"
             }
           ],
@@ -1728,7 +2100,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "GC duration ($instance)",
+          "title": "Slow inserts ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1744,7 +2116,96 @@
           },
           "yaxes": [
             {
-              "format": "s",
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "The number of slow loads of metric names during the last 5 minutes. If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series.\n\nUsually VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "hiddenSeries": false,
+          "id": 109,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(increase(vm_slow_metric_name_loads_total{job=\"$job_storage\", instance=~\"$instance\"}[5m]))",
+              "interval": "",
+              "legendFormat": "slow metrics load",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Slow metrics load ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1766,7 +2227,7 @@
           }
         }
       ],
-      "title": "Resource usage ($job)",
+      "title": "Troubleshooting",
       "type": "row"
     },
     {
@@ -1776,7 +2237,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 43
+        "y": 44
       },
       "id": 48,
       "panels": [
@@ -1793,7 +2254,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 4
           },
           "hiddenSeries": false,
           "id": 76,
@@ -1892,7 +2353,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 4
           },
           "hiddenSeries": false,
           "id": 86,
@@ -2005,7 +2466,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 53
+            "y": 13
           },
           "hiddenSeries": false,
           "id": 80,
@@ -2097,7 +2558,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 53
+            "y": 13
           },
           "hiddenSeries": false,
           "id": 78,
@@ -2189,7 +2650,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 61
+            "y": 21
           },
           "hiddenSeries": false,
           "id": 82,
@@ -2288,7 +2749,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 61
+            "y": 21
           },
           "hiddenSeries": false,
           "id": 84,
@@ -2377,7 +2838,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 69
+            "y": 29
           },
           "hiddenSeries": false,
           "id": 74,
@@ -2463,7 +2924,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 44
+        "y": 45
       },
       "id": 60,
       "panels": [
@@ -2480,8 +2941,9 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 45
+            "y": 13
           },
+          "hiddenSeries": false,
           "id": 57,
           "legend": {
             "alignAsTable": true,
@@ -2587,8 +3049,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 13
           },
+          "hiddenSeries": false,
           "id": 58,
           "legend": {
             "avg": false,
@@ -2676,7 +3139,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 45
+        "y": 46
       },
       "id": 24,
       "panels": [
@@ -2686,14 +3149,14 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "How many datapoints are in the storage.",
+          "description": "Shows how many datapoints are in the storage.",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 46
+            "y": 14
           },
           "hiddenSeries": false,
           "id": 16,
@@ -2723,8 +3186,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(vm_rows{job=\"$job_storage\", instance=~\"$instance\", type != \"indexdb\"}) by(instance)",
+              "expr": "sum(vm_rows{job=\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "refId": "A"
@@ -2758,6 +3222,7 @@
               "show": true
             },
             {
+              "decimals": null,
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -2784,7 +3249,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 46
+            "y": 14
           },
           "hiddenSeries": false,
           "id": 100,
@@ -2868,17 +3333,17 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "Shows amount of on-disk space occupied by inverted index.",
+          "description": "Shows amount of on-disk space occupied by data points.",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 54
+            "y": 22
           },
           "hiddenSeries": false,
-          "id": 20,
+          "id": 18,
           "legend": {
             "avg": false,
             "current": false,
@@ -2905,7 +3370,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}) by(instance)",
+              "expr": "sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
@@ -2916,7 +3381,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Disk space usage (index)  ($instance)",
+          "title": "Disk space usage (datapoints) ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -2959,17 +3424,17 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "Shows amount of on-disk space occupied by data points.",
+          "description": "Shows amount of on-disk space occupied by inverted index.",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 54
+            "y": 22
           },
           "hiddenSeries": false,
-          "id": 18,
+          "id": 20,
           "legend": {
             "avg": false,
             "current": false,
@@ -2996,7 +3461,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
+              "expr": "sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}) by(instance)",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
@@ -3007,7 +3472,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Disk space usage (datapoints) ($instance)",
+          "title": "Disk space usage (index)  ($instance)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -3057,7 +3522,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 62
+            "y": 30
           },
           "hiddenSeries": false,
           "id": 54,
@@ -3146,7 +3611,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 62
+            "y": 30
           },
           "hiddenSeries": false,
           "id": 14,
@@ -3254,7 +3719,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 70
+            "y": 38
           },
           "hiddenSeries": false,
           "id": 22,
@@ -3345,7 +3810,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 70
+            "y": 38
           },
           "hiddenSeries": false,
           "id": 55,
@@ -3434,7 +3899,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 78
+            "y": 46
           },
           "hiddenSeries": false,
           "id": 4,
@@ -3543,7 +4008,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 46
+        "y": 47
       },
       "id": 42,
       "panels": [
@@ -3560,7 +4025,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 47
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 92,
@@ -3654,7 +4119,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 47
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 95,
@@ -3765,7 +4230,7 @@
             "h": 8,
             "w": 24,
             "x": 0,
-            "y": 55
+            "y": 23
           },
           "hiddenSeries": false,
           "id": 93,
@@ -3870,7 +4335,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 47
+        "y": 48
       },
       "id": 40,
       "panels": [
@@ -3887,7 +4352,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 48
+            "y": 16
           },
           "hiddenSeries": false,
           "id": 97,
@@ -3981,7 +4446,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 48
+            "y": 16
           },
           "hiddenSeries": false,
           "id": 99,
@@ -4092,7 +4557,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 56
+            "y": 24
           },
           "hiddenSeries": false,
           "id": 90,
@@ -4199,7 +4664,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 56
+            "y": 24
           },
           "hiddenSeries": false,
           "id": 88,
@@ -4260,13 +4725,13 @@
           },
           "yaxes": [
             {
+              "decimals": 2,
               "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true,
-              "decimals": 2
+              "show": true
             },
             {
               "format": "short",
@@ -4294,7 +4759,7 @@
     "list": [
       {
         "current": {
-          "selected": true,
+          "selected": false,
           "text": "VictoriaMetrics",
           "value": "VictoriaMetrics"
         },
@@ -4432,7 +4897,7 @@
     ]
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-30m",
     "to": "now"
   },
   "timepicker": {
@@ -4456,4 +4921,4 @@
     "list": []
   },
   "version": 1
-}
+}
\ No newline at end of file
diff --git a/docs/Articles.md b/docs/Articles.md
index 91d8a319c..bfc1dc31d 100644
--- a/docs/Articles.md
+++ b/docs/Articles.md
@@ -34,3 +34,4 @@
 * [Disk usage: VictoriaMetrics vs Prometheus](https://stas.starikevich.com/posts/disk-usage-for-vm-versus-prometheus/)
 * [Benchmarking time series workloads on Apache Kudu using TSBS](https://blog.cloudera.com/benchmarking-time-series-workloads-on-apache-kudu-using-tsbs/)
 * [What are Open Source Time Series Databases?](https://www.iunera.com/kraken/fabric/time-series-database/)
+* [Evaluating performance and correctness](https://www.robustperception.io/evaluating-performance-and-correctness)


Test patch:
diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go
index afe30d5b7..1e96e6b49 100644
--- a/app/vmalert/manager_test.go
+++ b/app/vmalert/manager_test.go
@@ -26,24 +26,35 @@ func TestManagerUpdateError(t *testing.T) {
 // execution of configuration update.
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
-	m := &manager{groups: make(map[uint64]*Group)}
+	m := &manager{
+		groups:   make(map[uint64]*Group),
+		storage:  &fakeQuerier{},
+		notifier: &fakeNotifier{},
+	}
 	paths := []string{
 		"testdata/dir/rules0-good.rules",
 		"testdata/dir/rules1-good.rules",
 		"testdata/rules0-good.rules",
 	}
+	*evaluationInterval = time.Millisecond
+	if err := m.start(context.Background(), []string{paths[0]}, true); err != nil {
+		t.Fatalf("failed to start: %s", err)
+	}
 
-	const n = 500
+	const workers = 500
+	const iterations = 10
 	wg := sync.WaitGroup{}
-	wg.Add(n)
-	for i := 0; i < n; i++ {
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
 		go func() {
 			defer wg.Done()
-			rnd := rand.Intn(len(paths))
-			path := []string{paths[rnd]}
-			err := m.update(context.Background(), path, true, false)
-			if err != nil {
-				t.Errorf("update error: %s", err)
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				path := []string{paths[rnd]}
+				err := m.update(context.Background(), path, true, false)
+				if err != nil {
+					t.Errorf("update error: %s", err)
+				}
 			}
 		}()
 	}
@@ -111,7 +122,7 @@ func TestManagerUpdate(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group)}
+			m := &manager{groups: make(map[uint64]*Group), storage: &fakeQuerier{}}
 			path := []string{tc.initPath}
 			if err := m.update(ctx, path, true, false); err != nil {
 				t.Fatalf("failed to complete initial rules update: %s", err)
diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go
index 9d7cc3608..3583da05e 100644
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@@ -1,22 +1,27 @@
 package notifier
 
 import (
-	"fmt"
+	"net/url"
 	"testing"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	InitTemplateFunc(u)
 	testCases := []struct {
+		name string
 		alert       *Alert
 		annotations map[string]string
 		expTpl      map[string]string
 	}{
 		{
+			name: "empty-alert",
 			alert:       &Alert{},
 			annotations: map[string]string{},
 			expTpl:      map[string]string{},
 		},
 		{
+			name: "no-template",
 			alert: &Alert{
 				Value: 1e4,
 				Labels: map[string]string{
@@ -27,6 +32,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 			expTpl:      map[string]string{},
 		},
 		{
+			name: "label-template",
 			alert: &Alert{
 				Value: 1e4,
 				Labels: map[string]string{
@@ -43,10 +49,24 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"description": "It is 10000 connections for localhost",
 			},
 		},
+		{
+			name: "expression-template",
+			alert: &Alert{
+				Expr:  "vm_rows>0",
+			},
+			annotations: map[string]string{
+				"exprEscapedQuery": "{{ $expr|queryEscape }}",
+				"exprEscapedPath":  "{{ $expr|pathEscape }}",
+			},
+			expTpl: map[string]string{
+				"exprEscapedQuery": "vm_rows%3E0",
+				"exprEscapedPath":  "vm_rows%3E0",
+			},
+		},
 	}
 
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
 			tpl, err := tc.alert.ExecTemplate(tc.annotations)
 			if err != nil {
 				t.Fatal(err)
diff --git a/app/vmalert/rule_test.go b/app/vmalert/rule_test.go
index ae11105ba..73227b476 100644
--- a/app/vmalert/rule_test.go
+++ b/app/vmalert/rule_test.go
@@ -296,16 +296,14 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for-pending=>inactive", time.Millisecond),
+			newTestRule("for-pending=>empty", time.Second),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset pending alerts
+				// empty step to reset and delete pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
-			},
+			map[uint64]*notifier.Alert{},
 		},
 		{
 			newTestRule("for-pending=>firing=>inactive", time.Millisecond),
diff --git a/app/vmalert/testdata/dir/rules0-good.rules b/app/vmalert/testdata/dir/rules0-good.rules
index ec5b0bc9d..179858053 100644
--- a/app/vmalert/testdata/dir/rules0-good.rules
+++ b/app/vmalert/testdata/dir/rules0-good.rules
@@ -6,6 +6,7 @@ groups:
         expr: vm_rows > 0
         labels:
           label: bar
+          expr: "{{ $expr|queryEscape }}"
         annotations:
           summary: "{{ $value|humanize }}"
           description: "{{$labels}}"