Repository: VictoriaMetrics/VictoriaMetrics
Pull request #7394 (2024-10-30 15:04:22)

Add restarts charts
Reopening PR #7373 from a branch in VictoriaMetrics repo in order to enable edits and rebase. 

- [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).



Patch:
diff --git a/Makefile b/Makefile
index ead25d376..336a935bf 100644
--- a/Makefile
+++ b/Makefile
@@ -501,10 +501,12 @@ pprof-cpu:
 fmt:
 	gofmt -l -w -s ./lib
 	gofmt -l -w -s ./app
+	gofmt -l -w -s ./apptest
 
 vet:
 	go vet ./lib/...
 	go vet ./app/...
+	go vet ./apptest/...
 
 check-all: fmt vet golangci-lint govulncheck
 
@@ -525,6 +527,9 @@ test-full:
 test-full-386:
 	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
+integration-test: all
+	go test ./apptest/...
+
 benchmark:
 	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
diff --git a/dashboards/victorialogs.json b/dashboards/victorialogs.json
index 68c0d630a..de8cd19fe 100644
--- a/dashboards/victorialogs.json
+++ b/dashboards/victorialogs.json
@@ -1108,13 +1108,118 @@
       "title": "Log stream churn rate",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "id": 62,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "9.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Restarts ($job)",
+      "type": "timeseries"
+    },
     {
       "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 22
+        "y": 30
       },
       "id": 28,
       "panels": [
@@ -1168,8 +1273,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1185,7 +1289,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 11
           },
           "id": 38,
           "options": {
@@ -1275,8 +1379,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1292,7 +1395,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 11
           },
           "id": 40,
           "options": {
@@ -1432,8 +1535,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1449,7 +1551,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 42,
           "options": {
@@ -1538,8 +1640,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1555,7 +1656,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 44,
           "options": {
@@ -1648,8 +1749,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1681,7 +1781,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 46,
           "options": {
@@ -1773,8 +1873,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1806,7 +1905,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 48,
           "options": {
@@ -1911,8 +2010,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1928,7 +2026,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 50,
           "options": {
@@ -2017,8 +2115,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2047,7 +2144,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 52,
           "options": {
@@ -2150,8 +2247,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2167,7 +2263,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 54,
           "options": {
@@ -2256,8 +2352,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2286,7 +2381,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 56,
           "options": {
@@ -2393,8 +2488,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2410,7 +2504,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 58,
           "options": {
@@ -2501,8 +2595,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2518,7 +2611,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 51
           },
           "id": 60,
           "options": {
@@ -2610,8 +2703,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2627,7 +2719,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 59
           },
           "id": 61,
           "options": {
diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json
index df30817d0..bead53552 100644
--- a/dashboards/victoriametrics-cluster.json
+++ b/dashboards/victoriametrics-cluster.json
@@ -4124,6 +4124,111 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 70
+          },
+          "id": 214,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -9895,4 +10000,4 @@
   "uid": "oS7Bi_0Wz",
   "version": 1,
   "weekStart": ""
-}
\ No newline at end of file
+}
diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json
index 6210a460b..674eb3180 100644
--- a/dashboards/victoriametrics.json
+++ b/dashboards/victoriametrics.json
@@ -1569,7 +1569,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 11
           },
           "id": 112,
           "links": [],
@@ -1677,7 +1677,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 11
           },
           "id": 44,
           "options": {
@@ -1844,7 +1844,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 123,
           "links": [],
@@ -1951,7 +1951,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 114,
           "options": {
@@ -2044,8 +2044,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2077,7 +2076,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 75,
           "options": {
@@ -2169,8 +2168,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2202,7 +2200,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 57,
           "options": {
@@ -2309,8 +2307,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2326,7 +2323,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 47,
           "options": {
@@ -2415,8 +2412,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2445,7 +2441,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 76,
           "options": {
@@ -2552,8 +2548,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2569,7 +2564,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 48,
           "options": {
@@ -2658,8 +2653,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2688,7 +2682,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 124,
           "options": {
@@ -2795,8 +2789,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2812,7 +2805,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 49,
           "options": {
@@ -2903,8 +2896,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2920,7 +2912,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 51
           },
           "id": 37,
           "options": {
@@ -3010,8 +3002,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3040,7 +3031,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 59
           },
           "id": 127,
           "options": {
@@ -3146,8 +3137,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3163,7 +3153,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 59
           },
           "id": 125,
           "options": {
@@ -3253,8 +3243,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3270,7 +3259,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 128,
           "options": {
@@ -3386,7 +3375,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3394,8 +3384,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3403,7 +3392,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 4
+            "y": 32
           },
           "id": 66,
           "options": {
@@ -3505,7 +3494,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3513,8 +3503,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3522,10 +3511,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 4
+            "y": 32
           },
           "id": 68,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3615,7 +3603,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3623,8 +3612,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3632,10 +3620,9 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 12
+            "y": 40
           },
           "id": 116,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3723,7 +3710,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3731,8 +3719,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3740,10 +3727,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 12
+            "y": 40
           },
           "id": 60,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3830,7 +3816,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3838,8 +3825,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3847,7 +3833,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 48
           },
           "id": 90,
           "options": {
@@ -3938,7 +3924,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3946,8 +3933,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3955,10 +3941,9 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 48
           },
           "id": 118,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -4022,15 +4007,15 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
                     "value": 80
                   }
                 ]
-              },
-              "unitScale": true
+              }
             },
             "overrides": [
               {
@@ -4075,7 +4060,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 29
+            "y": 57
           },
           "id": 126,
           "options": {
@@ -4090,7 +4075,7 @@
             },
             "showHeader": true
           },
-          "pluginVersion": "10.3.1",
+          "pluginVersion": "10.4.2",
           "targets": [
             {
               "datasource": {
@@ -4161,7 +4146,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4169,8 +4155,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -4178,10 +4163,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 29
+            "y": 57
           },
           "id": 74,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -4218,6 +4202,111 @@
           ],
           "title": "Labels limit exceeded",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 65
+          },
+          "id": 129,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "targets": [
@@ -4313,7 +4402,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 5
+            "y": 13
           },
           "id": 10,
           "links": [],
@@ -4422,7 +4511,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 5
+            "y": 13
           },
           "id": 73,
           "links": [],
@@ -4532,7 +4621,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 21
           },
           "id": 53,
           "links": [],
@@ -4686,7 +4775,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 21
           },
           "id": 34,
           "links": [],
@@ -4824,7 +4913,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 21
+            "y": 29
           },
           "id": 30,
           "links": [],
@@ -4946,7 +5035,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 21
+            "y": 29
           },
           "id": 36,
           "links": [],
@@ -5054,7 +5143,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 29
+            "y": 37
           },
           "id": 58,
           "links": [],
@@ -5164,7 +5253,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 29
+            "y": 37
           },
           "id": 62,
           "options": {
@@ -5284,7 +5373,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 37
+            "y": 45
           },
           "id": 59,
           "links": [],
@@ -5404,7 +5493,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 37
+            "y": 45
           },
           "id": 64,
           "options": {
@@ -5510,7 +5599,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 45
+            "y": 53
           },
           "id": 99,
           "links": [],
@@ -5620,7 +5709,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 53
           },
           "id": 103,
           "links": [],
@@ -5730,7 +5819,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 53
+            "y": 61
           },
           "id": 122,
           "links": [],
@@ -5840,7 +5929,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 53
+            "y": 61
           },
           "id": 105,
           "links": [],
diff --git a/dashboards/vm/victoriametrics-cluster.json b/dashboards/vm/victoriametrics-cluster.json
index bbc000252..bbdcd6e47 100644
--- a/dashboards/vm/victoriametrics-cluster.json
+++ b/dashboards/vm/victoriametrics-cluster.json
@@ -4125,6 +4125,111 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 70
+          },
+          "id": 214,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -9896,4 +10001,4 @@
   "uid": "oS7Bi_0Wz_vm",
   "version": 1,
   "weekStart": ""
-}
\ No newline at end of file
+}
diff --git a/dashboards/vm/victoriametrics.json b/dashboards/vm/victoriametrics.json
index cbd11bfd3..7bf01e0f2 100644
--- a/dashboards/vm/victoriametrics.json
+++ b/dashboards/vm/victoriametrics.json
@@ -1570,7 +1570,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 11
           },
           "id": 112,
           "links": [],
@@ -1678,7 +1678,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 11
           },
           "id": 44,
           "options": {
@@ -1845,7 +1845,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 123,
           "links": [],
@@ -1952,7 +1952,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 114,
           "options": {
@@ -2045,8 +2045,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2078,7 +2077,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 75,
           "options": {
@@ -2170,8 +2169,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2203,7 +2201,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 57,
           "options": {
@@ -2310,8 +2308,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2327,7 +2324,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 47,
           "options": {
@@ -2416,8 +2413,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2446,7 +2442,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 76,
           "options": {
@@ -2553,8 +2549,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2570,7 +2565,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 48,
           "options": {
@@ -2659,8 +2654,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2689,7 +2683,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 124,
           "options": {
@@ -2796,8 +2790,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2813,7 +2806,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 49,
           "options": {
@@ -2904,8 +2897,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2921,7 +2913,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 51
           },
           "id": 37,
           "options": {
@@ -3011,8 +3003,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3041,7 +3032,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 59
           },
           "id": 127,
           "options": {
@@ -3147,8 +3138,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3164,7 +3154,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 59
           },
           "id": 125,
           "options": {
@@ -3254,8 +3244,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3271,7 +3260,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 128,
           "options": {
@@ -3387,7 +3376,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3395,8 +3385,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3404,7 +3393,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 4
+            "y": 32
           },
           "id": 66,
           "options": {
@@ -3506,7 +3495,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3514,8 +3504,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3523,10 +3512,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 4
+            "y": 32
           },
           "id": 68,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3616,7 +3604,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3624,8 +3613,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3633,10 +3621,9 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 12
+            "y": 40
           },
           "id": 116,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3724,7 +3711,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3732,8 +3720,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -3741,10 +3728,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 12
+            "y": 40
           },
           "id": 60,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -3831,7 +3817,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3839,8 +3826,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3848,7 +3834,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 48
           },
           "id": 90,
           "options": {
@@ -3939,7 +3925,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3947,8 +3934,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "unitScale": true
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -3956,10 +3942,9 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 48
           },
           "id": 118,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -4023,15 +4008,15 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
                     "value": 80
                   }
                 ]
-              },
-              "unitScale": true
+              }
             },
             "overrides": [
               {
@@ -4076,7 +4061,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 29
+            "y": 57
           },
           "id": 126,
           "options": {
@@ -4091,7 +4076,7 @@
             },
             "showHeader": true
           },
-          "pluginVersion": "10.3.1",
+          "pluginVersion": "10.4.2",
           "targets": [
             {
               "datasource": {
@@ -4162,7 +4147,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4170,8 +4156,7 @@
                   }
                 ]
               },
-              "unit": "short",
-              "unitScale": true
+              "unit": "short"
             },
             "overrides": []
           },
@@ -4179,10 +4164,9 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 29
+            "y": 57
           },
           "id": 74,
-          "links": [],
           "options": {
             "legend": {
               "calcs": [
@@ -4219,6 +4203,111 @@
           ],
           "title": "Labels limit exceeded",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 65
+          },
+          "id": 129,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "targets": [
@@ -4314,7 +4403,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 5
+            "y": 13
           },
           "id": 10,
           "links": [],
@@ -4423,7 +4512,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 5
+            "y": 13
           },
           "id": 73,
           "links": [],
@@ -4533,7 +4622,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 21
           },
           "id": 53,
           "links": [],
@@ -4687,7 +4776,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 21
           },
           "id": 34,
           "links": [],
@@ -4825,7 +4914,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 21
+            "y": 29
           },
           "id": 30,
           "links": [],
@@ -4947,7 +5036,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 21
+            "y": 29
           },
           "id": 36,
           "links": [],
@@ -5055,7 +5144,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 29
+            "y": 37
           },
           "id": 58,
           "links": [],
@@ -5165,7 +5254,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 29
+            "y": 37
           },
           "id": 62,
           "options": {
@@ -5285,7 +5374,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 37
+            "y": 45
           },
           "id": 59,
           "links": [],
@@ -5405,7 +5494,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 37
+            "y": 45
           },
           "id": 64,
           "options": {
@@ -5511,7 +5600,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 45
+            "y": 53
           },
           "id": 99,
           "links": [],
@@ -5621,7 +5710,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 53
           },
           "id": 103,
           "links": [],
@@ -5731,7 +5820,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 53
+            "y": 61
           },
           "id": 122,
           "links": [],
@@ -5841,7 +5930,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 53
+            "y": 61
           },
           "id": 105,
           "links": [],
diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json
index 9a64ed834..64cbb72ed 100644
--- a/dashboards/vm/vmagent.json
+++ b/dashboards/vm/vmagent.json
@@ -1336,8 +1336,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1440,8 +1439,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1619,7 +1617,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -1635,7 +1634,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 3
           },
           "id": 109,
           "options": {
@@ -1750,7 +1749,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 3
           },
           "id": 111,
           "options": {
@@ -1875,7 +1874,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 11
           },
           "id": 81,
           "options": {
@@ -2011,7 +2010,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 11
           },
           "id": 7,
           "options": {
@@ -2130,7 +2129,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 19
           },
           "id": 83,
           "options": {
@@ -2236,7 +2235,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 19
           },
           "id": 39,
           "options": {
@@ -2343,7 +2342,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 27
           },
           "id": 135,
           "options": {
@@ -2450,7 +2449,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 27
           },
           "id": 149,
           "options": {
@@ -2556,7 +2555,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 35
           },
           "id": 41,
           "options": {
@@ -2669,7 +2668,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2685,7 +2685,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 4
           },
           "id": 92,
           "options": {
@@ -2773,7 +2773,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2789,7 +2790,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 4
           },
           "id": 95,
           "options": {
@@ -2880,7 +2881,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2896,7 +2898,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 12
           },
           "id": 98,
           "options": {
@@ -2987,7 +2989,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3003,7 +3006,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 12
           },
           "id": 99,
           "options": {
@@ -3055,6 +3058,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3068,6 +3072,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3091,7 +3096,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3107,7 +3113,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 36
+            "y": 20
           },
           "id": 79,
           "options": {
@@ -3159,6 +3165,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3172,6 +3179,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3195,7 +3203,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3211,7 +3220,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 36
+            "y": 20
           },
           "id": 18,
           "links": [
@@ -3269,6 +3278,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3282,6 +3292,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3305,7 +3316,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3321,7 +3333,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 28
           },
           "id": 127,
           "options": {
@@ -3371,6 +3383,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3384,6 +3397,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3407,7 +3421,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3423,7 +3438,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 28
           },
           "id": 50,
           "options": {
@@ -3483,7 +3498,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3521,13 +3537,15 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 24,
+            "w": 12,
             "x": 0,
-            "y": 52
+            "y": 36
           },
           "id": 129,
           "options": {
+            "cellHeight": "sm",
             "footer": {
+              "countRows": false,
               "fields": "",
               "reducer": [
                 "sum"
@@ -3542,7 +3560,7 @@
               }
             ]
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "10.4.2",
           "targets": [
             {
               "datasource": {
@@ -3585,6 +3603,111 @@
             }
           ],
           "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 36
+          },
+          "id": 150,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "targets": [
@@ -3678,7 +3801,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 45
+            "y": 53
           },
           "id": 48,
           "options": {
@@ -3784,7 +3907,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 53
           },
           "id": 76,
           "options": {
@@ -3888,7 +4011,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 132,
           "options": {
@@ -3994,7 +4117,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 133,
           "options": {
@@ -4099,7 +4222,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 20,
           "options": {
@@ -4203,7 +4326,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 67
           },
           "id": 126,
           "options": {
@@ -4306,7 +4429,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 75
           },
           "id": 46,
           "options": {
@@ -4409,7 +4532,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 75
           },
           "id": 148,
           "options": {
@@ -4512,7 +4635,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 75
+            "y": 83
           },
           "id": 31,
           "options": {
@@ -4691,7 +4814,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 22
+            "y": 30
           },
           "id": 73,
           "options": {
@@ -4808,7 +4931,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 14
+            "y": 22
           },
           "id": 131,
           "options": {
@@ -4912,7 +5035,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 22
+            "y": 30
           },
           "id": 130,
           "options": {
@@ -5029,7 +5152,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 22
+            "y": 30
           },
           "id": 77,
           "options": {
@@ -5163,7 +5286,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 15
+            "y": 23
           },
           "id": 146,
           "options": {
@@ -5265,7 +5388,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 15
+            "y": 23
           },
           "id": 143,
           "options": {
@@ -5361,7 +5484,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 23
+            "y": 31
           },
           "id": 147,
           "options": {
@@ -5464,7 +5587,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 23
+            "y": 31
           },
           "id": 139,
           "options": {
@@ -5575,7 +5698,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 31
+            "y": 39
           },
           "id": 142,
           "options": {
@@ -5672,7 +5795,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 31
+            "y": 39
           },
           "id": 137,
           "options": {
@@ -5785,7 +5908,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 39
+            "y": 47
           },
           "id": 141,
           "options": {
@@ -5916,7 +6039,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 16
+            "y": 24
           },
           "id": 60,
           "options": {
@@ -6020,7 +6143,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 16
+            "y": 24
           },
           "id": 66,
           "options": {
@@ -6124,7 +6247,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 24
+            "y": 32
           },
           "id": 61,
           "options": {
@@ -6228,7 +6351,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 24
+            "y": 32
           },
           "id": 65,
           "options": {
@@ -6331,7 +6454,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 32
+            "y": 40
           },
           "id": 88,
           "options": {
@@ -6430,7 +6553,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 32
+            "y": 40
           },
           "id": 84,
           "options": {
@@ -6533,7 +6656,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 40
+            "y": 48
           },
           "id": 90,
           "options": {
@@ -6599,7 +6722,7 @@
             "h": 2,
             "w": 24,
             "x": 0,
-            "y": 25
+            "y": 33
           },
           "id": 115,
           "options": {
@@ -6677,7 +6800,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 119,
           "options": {
@@ -6781,7 +6904,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 117,
           "options": {
@@ -6887,7 +7010,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 125,
           "links": [
@@ -7009,7 +7132,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 123,
           "options": {
@@ -7139,7 +7262,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 121,
           "options": {
@@ -7212,8 +7335,8 @@
       {
         "current": {
           "selected": false,
-          "text": "VictoriaMetrics",
-          "value": "P4169E866C3094E38"
+          "text": "VictoriaMetrics - cluster",
+          "value": "PAF93674D0B4E9963"
         },
         "hide": 0,
         "includeAll": false,
@@ -7338,4 +7461,4 @@
   "uid": "G7Z9GzMGz_vm",
   "version": 1,
   "weekStart": ""
-}
+}
\ No newline at end of file
diff --git a/dashboards/vm/vmalert.json b/dashboards/vm/vmalert.json
index 3d71e5c18..61fa88537 100644
--- a/dashboards/vm/vmalert.json
+++ b/dashboards/vm/vmalert.json
@@ -1126,234 +1126,6 @@
       "title": "Rules execution errors ($instance)",
       "type": "timeseries"
     },
-    {
-      "datasource": {
-        "type": "victoriametrics-datasource",
-        "uid": "$ds"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "align": "auto",
-            "cellOptions": {
-              "type": "auto"
-            },
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Value"
-            },
-            "properties": [
-              {
-                "id": "custom.hidden",
-                "value": true
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Time"
-            },
-            "properties": [
-              {
-                "id": "custom.hidden",
-                "value": true
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 25
-      },
-      "id": 50,
-      "options": {
-        "footer": {
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": true,
-            "displayName": "job"
-          }
-        ]
-      },
-      "pluginVersion": "9.2.7",
-      "targets": [
-        {
-          "datasource": {
-            "type": "victoriametrics-datasource",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)",
-          "format": "table",
-          "instant": true,
-          "legendFormat": "__auto",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Non-default flags",
-      "transformations": [
-        {
-          "id": "groupBy",
-          "options": {
-            "fields": {
-              "instance": {
-                "aggregations": [
-                  "uniqueValues"
-                ],
-                "operation": "aggregate"
-              },
-              "job": {
-                "aggregations": [],
-                "operation": "groupby"
-              },
-              "name": {
-                "aggregations": [],
-                "operation": "groupby"
-              },
-              "value": {
-                "aggregations": [],
-                "operation": "groupby"
-              }
-            }
-          }
-        }
-      ],
-      "type": "table"
-    },
-    {
-      "datasource": {
-        "type": "victoriametrics-datasource",
-        "uid": "$ds"
-      },
-      "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries.\"",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "bars",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 25
-      },
-      "id": 58,
-      "options": {
-        "legend": {
-          "calcs": [
-            "mean",
-            "lastNotNull",
-            "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "9.2.6",
-      "targets": [
-        {
-          "datasource": {
-            "type": "victoriametrics-datasource",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
-          "interval": "1m",
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Missed evaluations ($instance)",
-      "type": "timeseries"
-    },
     {
       "collapsed": true,
       "datasource": {
@@ -1364,7 +1136,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 32
+        "y": 25
       },
       "id": 43,
       "panels": [
@@ -1435,7 +1207,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 11
           },
           "id": 37,
           "links": [
@@ -1532,8 +1304,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1549,7 +1320,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 11
           },
           "id": 57,
           "links": [
@@ -1663,7 +1434,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 35,
           "links": [
@@ -1779,7 +1550,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 56,
           "links": [
@@ -1912,7 +1683,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 39,
           "links": [],
@@ -2020,7 +1791,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 41,
           "links": [],
@@ -2128,7 +1899,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 59,
           "links": [],
@@ -2219,8 +1990,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2236,7 +2006,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 61,
           "options": {
@@ -2288,6 +2058,359 @@
       "title": "Resource usage",
       "type": "row"
     },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 62,
+      "panels": [
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "auto",
+                "cellOptions": {
+                  "type": "auto"
+                },
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Value"
+                },
+                "properties": [
+                  {
+                    "id": "custom.hidden",
+                    "value": true
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Time"
+                },
+                "properties": [
+                  {
+                    "id": "custom.hidden",
+                    "value": true
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 50,
+          "options": {
+            "cellHeight": "sm",
+            "footer": {
+              "countRows": false,
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "job"
+              }
+            ]
+          },
+          "pluginVersion": "10.4.2",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)",
+              "format": "table",
+              "instant": true,
+              "legendFormat": "__auto",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "Non-default flags",
+          "transformations": [
+            {
+              "id": "groupBy",
+              "options": {
+                "fields": {
+                  "instance": {
+                    "aggregations": [
+                      "uniqueValues"
+                    ],
+                    "operation": "aggregate"
+                  },
+                  "job": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  },
+                  "name": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  },
+                  "value": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  }
+                }
+              }
+            }
+          ],
+          "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries.\"",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "id": 58,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
+              "interval": "1m",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Missed evaluations ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 34
+          },
+          "id": 63,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Troubleshooting",
+      "type": "row"
+    },
     {
       "collapsed": true,
       "datasource": {
@@ -2298,7 +2421,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 33
+        "y": 27
       },
       "id": 17,
       "panels": [
@@ -2364,7 +2487,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 52
           },
           "id": 14,
           "options": {
@@ -2466,7 +2589,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 52
           },
           "id": 13,
           "options": {
@@ -2568,7 +2691,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 20,
           "options": {
@@ -2671,7 +2794,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 32,
           "options": {
@@ -2770,7 +2893,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 60
+            "y": 68
           },
           "id": 26,
           "options": {
@@ -2831,7 +2954,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 34
+        "y": 28
       },
       "id": 28,
       "panels": [
@@ -2897,7 +3020,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -2999,7 +3122,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 67
           },
           "id": 33,
           "options": {
@@ -3100,7 +3223,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 75
           },
           "id": 30,
           "options": {
@@ -3157,7 +3280,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 35
+        "y": 29
       },
       "id": 55,
       "panels": [
@@ -3221,7 +3344,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 25
+            "y": 33
           },
           "id": 52,
           "options": {
@@ -3313,7 +3436,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 25
+            "y": 33
           },
           "id": 53,
           "options": {
@@ -3410,7 +3533,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 54,
           "options": {
@@ -3513,7 +3636,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 60,
           "options": {
@@ -3566,8 +3689,8 @@
       {
         "current": {
           "selected": false,
-          "text": "VictoriaMetrics",
-          "value": "P4169E866C3094E38"
+          "text": "VictoriaMetrics - cluster",
+          "value": "PAF93674D0B4E9963"
         },
         "hide": 0,
         "includeAll": false,
diff --git a/dashboards/vm/vmauth.json b/dashboards/vm/vmauth.json
index b4a7d4012..5bc9d8b5b 100644
--- a/dashboards/vm/vmauth.json
+++ b/dashboards/vm/vmauth.json
@@ -1209,8 +1209,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1226,7 +1225,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 25,
           "options": {
@@ -1315,8 +1314,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1332,7 +1330,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 26,
           "options": {
@@ -1424,8 +1422,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1441,7 +1438,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 27,
           "options": {
@@ -1581,8 +1578,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1614,7 +1610,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 28,
           "options": {
@@ -1717,8 +1713,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1734,7 +1729,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 23,
           "options": {
@@ -1823,8 +1818,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1840,7 +1834,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 24,
           "options": {
@@ -1932,8 +1926,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1965,7 +1958,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 20,
           "options": {
@@ -2057,8 +2050,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2074,7 +2066,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 21,
           "options": {
@@ -2162,8 +2154,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2179,7 +2170,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 29,
           "options": {
@@ -2453,6 +2444,111 @@
           ],
           "title": "Log errors",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 47
+          },
+          "id": 37,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json
index 8fa8b9acd..1f2fb0312 100644
--- a/dashboards/vmagent.json
+++ b/dashboards/vmagent.json
@@ -1335,8 +1335,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1439,8 +1438,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1618,7 +1616,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -1634,7 +1633,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 3
           },
           "id": 109,
           "options": {
@@ -1749,7 +1748,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 3
           },
           "id": 111,
           "options": {
@@ -1874,7 +1873,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 11
           },
           "id": 81,
           "options": {
@@ -2010,7 +2009,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 11
           },
           "id": 7,
           "options": {
@@ -2129,7 +2128,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 19
           },
           "id": 83,
           "options": {
@@ -2235,7 +2234,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 19
           },
           "id": 39,
           "options": {
@@ -2342,7 +2341,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 27
           },
           "id": 135,
           "options": {
@@ -2449,7 +2448,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 27
           },
           "id": 149,
           "options": {
@@ -2555,7 +2554,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 35
           },
           "id": 41,
           "options": {
@@ -2668,7 +2667,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2684,7 +2684,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 4
           },
           "id": 92,
           "options": {
@@ -2772,7 +2772,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2788,7 +2789,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 4
           },
           "id": 95,
           "options": {
@@ -2879,7 +2880,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -2895,7 +2897,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 12
           },
           "id": 98,
           "options": {
@@ -2986,7 +2988,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent"
+                    "color": "transparent",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3002,7 +3005,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 12
           },
           "id": 99,
           "options": {
@@ -3054,6 +3057,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3067,6 +3071,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3090,7 +3095,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3106,7 +3112,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 36
+            "y": 20
           },
           "id": 79,
           "options": {
@@ -3158,6 +3164,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3171,6 +3178,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3194,7 +3202,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3210,7 +3219,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 36
+            "y": 20
           },
           "id": 18,
           "links": [
@@ -3268,6 +3277,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3281,6 +3291,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3304,7 +3315,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3320,7 +3332,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 28
           },
           "id": 127,
           "options": {
@@ -3370,6 +3382,7 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
@@ -3383,6 +3396,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3406,7 +3420,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3422,7 +3437,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 28
           },
           "id": 50,
           "options": {
@@ -3482,7 +3497,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3520,13 +3536,15 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 24,
+            "w": 12,
             "x": 0,
-            "y": 52
+            "y": 36
           },
           "id": 129,
           "options": {
+            "cellHeight": "sm",
             "footer": {
+              "countRows": false,
               "fields": "",
               "reducer": [
                 "sum"
@@ -3541,7 +3559,7 @@
               }
             ]
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "10.4.2",
           "targets": [
             {
               "datasource": {
@@ -3584,6 +3602,111 @@
             }
           ],
           "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 36
+          },
+          "id": 150,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "targets": [
@@ -3677,7 +3800,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 45
+            "y": 53
           },
           "id": 48,
           "options": {
@@ -3783,7 +3906,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 53
           },
           "id": 76,
           "options": {
@@ -3887,7 +4010,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 132,
           "options": {
@@ -3993,7 +4116,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 133,
           "options": {
@@ -4098,7 +4221,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 20,
           "options": {
@@ -4202,7 +4325,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 67
           },
           "id": 126,
           "options": {
@@ -4305,7 +4428,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 75
           },
           "id": 46,
           "options": {
@@ -4408,7 +4531,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 75
           },
           "id": 148,
           "options": {
@@ -4511,7 +4634,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 75
+            "y": 83
           },
           "id": 31,
           "options": {
@@ -4690,7 +4813,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 22
+            "y": 30
           },
           "id": 73,
           "options": {
@@ -4807,7 +4930,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 14
+            "y": 22
           },
           "id": 131,
           "options": {
@@ -4911,7 +5034,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 22
+            "y": 30
           },
           "id": 130,
           "options": {
@@ -5028,7 +5151,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 22
+            "y": 30
           },
           "id": 77,
           "options": {
@@ -5162,7 +5285,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 15
+            "y": 23
           },
           "id": 146,
           "options": {
@@ -5264,7 +5387,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 15
+            "y": 23
           },
           "id": 143,
           "options": {
@@ -5360,7 +5483,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 23
+            "y": 31
           },
           "id": 147,
           "options": {
@@ -5463,7 +5586,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 23
+            "y": 31
           },
           "id": 139,
           "options": {
@@ -5574,7 +5697,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 31
+            "y": 39
           },
           "id": 142,
           "options": {
@@ -5671,7 +5794,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 31
+            "y": 39
           },
           "id": 137,
           "options": {
@@ -5784,7 +5907,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 39
+            "y": 47
           },
           "id": 141,
           "options": {
@@ -5915,7 +6038,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 16
+            "y": 24
           },
           "id": 60,
           "options": {
@@ -6019,7 +6142,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 16
+            "y": 24
           },
           "id": 66,
           "options": {
@@ -6123,7 +6246,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 24
+            "y": 32
           },
           "id": 61,
           "options": {
@@ -6227,7 +6350,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 24
+            "y": 32
           },
           "id": 65,
           "options": {
@@ -6330,7 +6453,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 32
+            "y": 40
           },
           "id": 88,
           "options": {
@@ -6429,7 +6552,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 32
+            "y": 40
           },
           "id": 84,
           "options": {
@@ -6532,7 +6655,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 40
+            "y": 48
           },
           "id": 90,
           "options": {
@@ -6598,7 +6721,7 @@
             "h": 2,
             "w": 24,
             "x": 0,
-            "y": 25
+            "y": 33
           },
           "id": 115,
           "options": {
@@ -6676,7 +6799,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 119,
           "options": {
@@ -6780,7 +6903,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 117,
           "options": {
@@ -6886,7 +7009,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 125,
           "links": [
@@ -7008,7 +7131,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 123,
           "options": {
@@ -7138,7 +7261,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 121,
           "options": {
@@ -7211,8 +7334,8 @@
       {
         "current": {
           "selected": false,
-          "text": "VictoriaMetrics",
-          "value": "P4169E866C3094E38"
+          "text": "VictoriaMetrics - cluster",
+          "value": "PAF93674D0B4E9963"
         },
         "hide": 0,
         "includeAll": false,
@@ -7337,4 +7460,4 @@
   "uid": "G7Z9GzMGz",
   "version": 1,
   "weekStart": ""
-}
+}
\ No newline at end of file
diff --git a/dashboards/vmalert.json b/dashboards/vmalert.json
index 405f438b0..929b9b2bb 100644
--- a/dashboards/vmalert.json
+++ b/dashboards/vmalert.json
@@ -1125,234 +1125,6 @@
       "title": "Rules execution errors ($instance)",
       "type": "timeseries"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "align": "auto",
-            "cellOptions": {
-              "type": "auto"
-            },
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Value"
-            },
-            "properties": [
-              {
-                "id": "custom.hidden",
-                "value": true
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Time"
-            },
-            "properties": [
-              {
-                "id": "custom.hidden",
-                "value": true
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 25
-      },
-      "id": 50,
-      "options": {
-        "footer": {
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": true,
-            "displayName": "job"
-          }
-        ]
-      },
-      "pluginVersion": "9.2.7",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)",
-          "format": "table",
-          "instant": true,
-          "legendFormat": "__auto",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Non-default flags",
-      "transformations": [
-        {
-          "id": "groupBy",
-          "options": {
-            "fields": {
-              "instance": {
-                "aggregations": [
-                  "uniqueValues"
-                ],
-                "operation": "aggregate"
-              },
-              "job": {
-                "aggregations": [],
-                "operation": "groupby"
-              },
-              "name": {
-                "aggregations": [],
-                "operation": "groupby"
-              },
-              "value": {
-                "aggregations": [],
-                "operation": "groupby"
-              }
-            }
-          }
-        }
-      ],
-      "type": "table"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds"
-      },
-      "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries.\"",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "bars",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 25
-      },
-      "id": 58,
-      "options": {
-        "legend": {
-          "calcs": [
-            "mean",
-            "lastNotNull",
-            "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "9.2.6",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
-          "interval": "1m",
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Missed evaluations ($instance)",
-      "type": "timeseries"
-    },
     {
       "collapsed": true,
       "datasource": {
@@ -1363,7 +1135,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 32
+        "y": 25
       },
       "id": 43,
       "panels": [
@@ -1434,7 +1206,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 11
           },
           "id": 37,
           "links": [
@@ -1531,8 +1303,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1548,7 +1319,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 11
           },
           "id": 57,
           "links": [
@@ -1662,7 +1433,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 35,
           "links": [
@@ -1778,7 +1549,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 56,
           "links": [
@@ -1911,7 +1682,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 39,
           "links": [],
@@ -2019,7 +1790,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 41,
           "links": [],
@@ -2127,7 +1898,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 59,
           "links": [],
@@ -2218,8 +1989,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2235,7 +2005,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 61,
           "options": {
@@ -2287,6 +2057,359 @@
       "title": "Resource usage",
       "type": "row"
     },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 62,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "auto",
+                "cellOptions": {
+                  "type": "auto"
+                },
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Value"
+                },
+                "properties": [
+                  {
+                    "id": "custom.hidden",
+                    "value": true
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Time"
+                },
+                "properties": [
+                  {
+                    "id": "custom.hidden",
+                    "value": true
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 50,
+          "options": {
+            "cellHeight": "sm",
+            "footer": {
+              "countRows": false,
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "job"
+              }
+            ]
+          },
+          "pluginVersion": "10.4.2",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)",
+              "format": "table",
+              "instant": true,
+              "legendFormat": "__auto",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "Non-default flags",
+          "transformations": [
+            {
+              "id": "groupBy",
+              "options": {
+                "fields": {
+                  "instance": {
+                    "aggregations": [
+                      "uniqueValues"
+                    ],
+                    "operation": "aggregate"
+                  },
+                  "job": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  },
+                  "name": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  },
+                  "value": {
+                    "aggregations": [],
+                    "operation": "groupby"
+                  }
+                }
+              }
+            }
+          ],
+          "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/troubleshooting/#slow-queries.\"",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "id": 58,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
+              "interval": "1m",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Missed evaluations ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 34
+          },
+          "id": 63,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Troubleshooting",
+      "type": "row"
+    },
     {
       "collapsed": true,
       "datasource": {
@@ -2297,7 +2420,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 33
+        "y": 27
       },
       "id": 17,
       "panels": [
@@ -2363,7 +2486,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 52
           },
           "id": 14,
           "options": {
@@ -2465,7 +2588,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 52
           },
           "id": 13,
           "options": {
@@ -2567,7 +2690,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 20,
           "options": {
@@ -2670,7 +2793,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 32,
           "options": {
@@ -2769,7 +2892,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 60
+            "y": 68
           },
           "id": 26,
           "options": {
@@ -2830,7 +2953,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 34
+        "y": 28
       },
       "id": 28,
       "panels": [
@@ -2896,7 +3019,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -2998,7 +3121,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 67
           },
           "id": 33,
           "options": {
@@ -3099,7 +3222,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 75
           },
           "id": 30,
           "options": {
@@ -3156,7 +3279,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 35
+        "y": 29
       },
       "id": 55,
       "panels": [
@@ -3220,7 +3343,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 25
+            "y": 33
           },
           "id": 52,
           "options": {
@@ -3312,7 +3435,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 25
+            "y": 33
           },
           "id": 53,
           "options": {
@@ -3409,7 +3532,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 60
           },
           "id": 54,
           "options": {
@@ -3512,7 +3635,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 60
           },
           "id": 60,
           "options": {
@@ -3565,8 +3688,8 @@
       {
         "current": {
           "selected": false,
-          "text": "VictoriaMetrics",
-          "value": "P4169E866C3094E38"
+          "text": "VictoriaMetrics - cluster",
+          "value": "PAF93674D0B4E9963"
         },
         "hide": 0,
         "includeAll": false,
diff --git a/dashboards/vmauth.json b/dashboards/vmauth.json
index c34ef9a8b..e81692876 100644
--- a/dashboards/vmauth.json
+++ b/dashboards/vmauth.json
@@ -1208,8 +1208,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1225,7 +1224,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 19
           },
           "id": 25,
           "options": {
@@ -1314,8 +1313,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1331,7 +1329,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 19
           },
           "id": 26,
           "options": {
@@ -1423,8 +1421,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1440,7 +1437,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 27
           },
           "id": 27,
           "options": {
@@ -1580,8 +1577,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1613,7 +1609,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 27
           },
           "id": 28,
           "options": {
@@ -1716,8 +1712,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1733,7 +1728,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 35
           },
           "id": 23,
           "options": {
@@ -1822,8 +1817,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1839,7 +1833,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 35
           },
           "id": 24,
           "options": {
@@ -1931,8 +1925,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1964,7 +1957,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 43
           },
           "id": 20,
           "options": {
@@ -2056,8 +2049,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2073,7 +2065,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 43
           },
           "id": 21,
           "options": {
@@ -2161,8 +2153,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2178,7 +2169,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 51
           },
           "id": 29,
           "options": {
@@ -2452,6 +2443,111 @@
           ],
           "title": "Log errors",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "stepAfter",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 47
+          },
+          "id": 37,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.1.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Restarts ($job)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
diff --git a/deployment/docker/README.md b/deployment/docker/README.md
index d9e136af7..bdb86697a 100644
--- a/deployment/docker/README.md
+++ b/deployment/docker/README.md
@@ -165,7 +165,7 @@ The list of alerting rules is the following:
   alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert/) component;
 * [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml):
   alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component;
-* [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vlogs.yml):
+* [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml):
     alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/);
 
 Please, also see [how to monitor](https://docs.victoriametrics.com/single-server-victoriametrics/#monitoring) 
diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml
index 88754ac96..c9027657c 100644
--- a/deployment/docker/docker-compose-victorialogs.yml
+++ b/deployment/docker/docker-compose-victorialogs.yml
@@ -99,9 +99,8 @@ services:
     ports:
       - 8880:8880
     volumes:
-      # disable log-related rules for now, util vmalert supports vlogs type rule
-      # - ./rules/vlogs-example-alerts.yml:/etc/alerts/vlogs.yml
       - ./rules/alerts.yml:/etc/alerts/alerts.yml
+      - ./rules/alerts-vlogs.yml:/etc/alerts/vlogs.yml
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
       - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
diff --git a/deployment/docker/alerts-vlogs.yml b/deployment/docker/rules/alerts-vlogs.yml
similarity index 100%
rename from deployment/docker/alerts-vlogs.yml
rename to deployment/docker/rules/alerts-vlogs.yml
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index 99d5726d1..5f6da4947 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -41,7 +41,7 @@ Released at 2024-10-29
 Released at 2024-10-18
 
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add ability to hide hits chart. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7117).
-* FEATURE: add basic [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vlogs.yml) for VictoriaLogs process. See details at [monitoring docs](https://docs.victoriametrics.com/victorialogs/#monitoring).
+* FEATURE: add basic [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml) for VictoriaLogs process. See details at [monitoring docs](https://docs.victoriametrics.com/victorialogs/#monitoring).
 * FEATURE: improve [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) performance on systems with many CPU cores when `by(...)` fields contain big number of unique values. For example, `_time:1d | stats by (user_id) count() x` should be executed much faster when `user_id` field contains millions of unique values.
 * FEATURE: improve performance for [`top`](https://docs.victoriametrics.com/victorialogs/logsql/#top-pipe), [`uniq`](https://docs.victoriametrics.com/victorialogs/logsql/#uniq-pipe) and [`field_values`](https://docs.victoriametrics.com/victorialogs/logsql/#field_values-pipe) pipes on systems with many CPU cores when it is applied to [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) with big number of unique values. For example, `_time:1d | top 5 (user_id)` should be executed much faster when `user_id` field contains millions of unique values.
 * FEATURE: improve performance for [`field_names` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#field_names-pipe) when it is applied to logs with hundreds of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
diff --git a/docs/VictoriaLogs/README.md b/docs/VictoriaLogs/README.md
index b37fc242c..1b0dbf2c8 100644
--- a/docs/VictoriaLogs/README.md
+++ b/docs/VictoriaLogs/README.md
@@ -52,7 +52,7 @@ It is recommended to set up monitoring of these metrics via VictoriaMetrics
 (see [these docs](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter)),
 vmagent (see [these docs](https://docs.victoriametrics.com/vmagent/#how-to-collect-metrics-in-prometheus-format)) or via Prometheus.
 
-We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vlogs.yml)
+We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml)
 via [vmalert](https://docs.victoriametrics.com/vmalert/) or via Prometheus.
 
 VictoriaLogs emits its own logs to stdout. It is recommended to investigate these logs during troubleshooting.
diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md
index 2b9d9b811..551f983db 100644
--- a/docs/changelog/CHANGELOG.md
+++ b/docs/changelog/CHANGELOG.md
@@ -22,6 +22,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support scraping from Kubernetes Native Sidecars. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287).
 * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add a separate cache type for storing sparse entries when performing large index scans. This significantly reduces memory usage when applying [downsampling filters](https://docs.victoriametrics.com/#downsampling) and [retention filters](https://docs.victoriametrics.com/#retention-filters) during background merge. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7182) for the details.
+* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for VM single-node, cluster, vmalert, vmagent, VictoriaLogs: add `Restarts` panel to show the events of process restarts. This panel should help correlate events of restart with unexpected behavior of processes.
 
 * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for Single-node VictoriaMetrics, cluster: The free disk space calculation now will subtract the size of the `-storage.minFreeDiskSpaceBytes` flag to correctly display the remaining available space of Single-node VictoriaMetrics/vmstorage rather than the actual available disk space, as well as the full ETA. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7334) for the details.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly set `group_name` and `file` fields for recording rules in `/api/v1/rules`.
@@ -81,7 +82,7 @@ Released at 2024-10-02
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): keep selected columns in table view on page reloads. Before, selected columns were reset on each update. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7016).
 * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for VM single-node, cluster, vmalert, vmagent, VictoriaLogs: add `Go scheduling latency` panel to show the 99th quantile of Go goroutines scheduling. This panel should help identifying insufficient CPU resources for the service. It is especially useful if CPU gets throttled, which now should be visible on this panel.
 * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): update  [dashboard for VictoriaMetrics k8s operator](https://grafana.com/grafana/dashboards/17869-victoriametrics-operator/): replace deprecated TimeSeries panels, add latency panels for rest client and Golang scheduler, add overview panels. The dashboard is now also available for [VictoriaMetrics Grafana datasource](https://github.com/VictoriaMetrics/victoriametrics-datasource).
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): add alerting rule to track the Go scheduling latency for goroutines. It should notify users if VM component doesn't have enough CPU to run or gets throttled.
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): add alerting rule to track the Go scheduling latency for goroutines. It should notify users if VM component doesn't have enough CPU to run or gets throttled.
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink` in `Host:Port` format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly consume messages [from kafka](https://docs.victoriametrics.com/vmagent/#kafka-integration). Previously vmagent could skip some messages during start-up.
@@ -271,7 +272,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/) and [vmrestore](https://docs.victoriametrics.com/vmrestore/): add support for [Azure Managed Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) and default credentials lookup. See [these docs](https://docs.victoriametrics.com/vmbackup/#providing-credentials-via-env-variables) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5984) for the details. Thanks to @justinrush for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6518).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/) and [vmrestore](https://docs.victoriametrics.com/vmbackup/): allow overriding Azure storage domain when performing backups via `AZURE_STORAGE_DOMAIN` environment variable. See [these docs](https://docs.victoriametrics.com/vmbackup/#providing-credentials-via-env-variables) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5984). Thanks to @justinrush for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6518).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show compacted result in the JSON tab for query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6559).
-* FEATURE: [dashboards](https://grafana.com/orgs/victoriametrics): add [Grafana dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmauth.json) and [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmauth.yml) for [vmauth](https://docs.victoriametrics.com/vmauth/) dashboard. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4313) for details.
+* FEATURE: [dashboards](https://grafana.com/orgs/victoriametrics): add [Grafana dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmauth.json) and [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml) for [vmauth](https://docs.victoriametrics.com/vmauth/) dashboard. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4313) for details.
 * FEATURE: [vmagent dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmagent.json): `stream aggregation` section: add graphs based on newly exposed streaming aggregation metrics.
 
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): properly calculate [histogram_quantile](https://docs.victoriametrics.com/metricsql/#histogram_quantile) over Prometheus buckets with inconsistent values. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6547). Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2819).
@@ -290,7 +291,7 @@ Released at 2024-06-24
 
 **Update note 2: `*.passwordFile` and similar flags are no longer trimming trailing whitespaces at the end of content. Make sure to update the templating of password files or HTTP endpoints to not include trailing whitespaces before the upgrade. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6503) PR for the details.**
 
-* FEATURE: [alerts-vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): add new alerting rules `StreamAggrFlushTimeout` and `StreamAggrDedupFlushTimeout` to notify about issues during stream aggregation.
+* FEATURE: [alerts-vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmagent.yml): add new alerting rules `StreamAggrFlushTimeout` and `StreamAggrDedupFlushTimeout` to notify about issues during stream aggregation.
 * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add row `Streaming aggregation` with panels related to [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation/) process.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): add `-idleConnTimeout` flag set to 50s by default. It should reduce the probability of `broken pipe` or `connection reset by peer` errors in vmauth logs.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): add automatic retry for requests to backend for trivial network errors, such as `broken pipe` and `connection reset` for requests to the configured backends.
@@ -347,8 +348,8 @@ Released at 2024-06-07
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): speed up retrieving rules files from object storages by skipping unchanged objects during reloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6210).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support reading [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) records in `-datasource.url`, `-remoteWrite.url` and `-remoteRead.url` command-line option. For example, `-remoteWrite.url=http://srv+victoria-metrics` automatically resolves the `victoria-metrics` DNS SRV to a list of hostnames with TCP ports and then sends data to one of the addresses. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6053).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): add `-s3TLSInsecureSkipVerify` command-line flag for skipping TLS certificates verification when connecting to S3 endpoint.
-* FEATURE: expose metric `vm_indexdb_items_dropped_total` to track the number of IndexDB records that had to be dropped during ingestion. The reason of dropping the record will be annotated in `reason` label of the exposed metric. This change also comes with a new [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) to track changes of this metric.
-* FEATURE: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): add new alerting rules `TooLongLabelValues` and `TooLongLabelNames` to notify about truncation of label values or names respectively.
+* FEATURE: expose metric `vm_indexdb_items_dropped_total` to track the number of IndexDB records that had to be dropped during ingestion. The reason of dropping the record will be annotated in `reason` label of the exposed metric. This change also comes with a new [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) to track changes of this metric.
+* FEATURE: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): add new alerting rules `TooLongLabelValues` and `TooLongLabelNames` to notify about truncation of label values or names respectively.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): expose `vm_streamaggr_ignored_samples_total` [counters](https://docs.victoriametrics.com/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/#monitoring), which can be used for detecting amount of too old or NaN valued ignored samples. Expose also `vm_streamaggr_samples_lag_seconds` [histogram](https://docs.victoriametrics.com/keyconcepts/#histogram) to monitor aggregated samples lag.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): improve filtering speed of the received data samples if [match](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config) field is matching only [metric name](https://docs.victoriametrics.com/keyconcepts/#structure-of-a-metric).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): adds `-relabelConfigCheckInterval` flag and `/-/reload` endpoint for reloading relabeling rules. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3923) for details.
diff --git a/docs/changelog/CHANGELOG_2022.md b/docs/changelog/CHANGELOG_2022.md
index 9dd33b57d..9364dca72 100644
--- a/docs/changelog/CHANGELOG_2022.md
+++ b/docs/changelog/CHANGELOG_2022.md
@@ -147,7 +147,7 @@ Released at 2022-11-25
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add compact table view. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3241).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to "stick" a tooltip on the chart by clicking on a data point. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3321) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3376)
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to set up series custom limits. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3297).
-* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): add default alert list for vmalert's metrics. See [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmalert.yml).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): add default alert list for vmalert's metrics. See [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): expose `vmagent_relabel_config_*`, `vm_relabel_config_*` and `vm_promscrape_config_*` metrics for tracking relabel and scrape configuration hot-reloads. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3345).
 
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): properly return an empty result from [limit_offset](https://docs.victoriametrics.com/metricsql/#limit_offset) if the `offset` arg exceeds the number of inner time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3312).
diff --git a/docs/changelog/CHANGELOG_2023.md b/docs/changelog/CHANGELOG_2023.md
index 3db999e61..319c96f9c 100644
--- a/docs/changelog/CHANGELOG_2023.md
+++ b/docs/changelog/CHANGELOG_2023.md
@@ -364,7 +364,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dashboard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit.
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627).
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
-* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. 
+* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) as it could be related to many VictoriaMetrics components. 
 
 * BUGFIX: [storage](https://docs.victoriametrics.com/single-server-victoriametrics/): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): return human readable error if opentelemetry has json encoding. Follow-up after [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2570).
@@ -564,7 +564,7 @@ Released at 2023-05-18
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the display of the tenant selector. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4160).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix UI freeze when the query returns non-histogram series alongside histogram series.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the text display on buttons in Safari 16.4.
-* BUGFIX: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): update threshold for `TooHighMemoryUsage` alert from 90% to 80%, since 90% is too high for production environments.
+* BUGFIX: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): update threshold for `TooHighMemoryUsage` alert from 90% to 80%, since 90% is too high for production environments.
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/): fix compatibility with Windows OS. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl/#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): fix bug with duplicated labels during stream aggregation via single-node VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4277).
@@ -827,7 +827,7 @@ The v1.87.x line will be supported for at least 12 months since [v1.87.0](https:
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): add `__meta_kubernetes_endpoints_name` label for all ports discovered from endpoint. Previously, ports not matched by `Service` did not have this label. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4154) for details. Thanks to @thunderbird86 for discovering and [fixing](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4253) the issue.
 * BUGFIX: fix possible infinite loop during `indexdb` rotation when `-retentionTimezoneOffset` command-line flag is set and the local timezone is not UTC. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4207). Thanks to @faceair for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4206).
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): do not return invalid auth credentials in http response by default, since it may be logged by client. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4188).
-* BUGFIX: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): update threshold for `TooHighMemoryUsage` alert from 90% to 80%, since 90% is too high for production environments.
+* BUGFIX: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): update threshold for `TooHighMemoryUsage` alert from 90% to 80%, since 90% is too high for production environments.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly handle the `vm_promscrape_config_last_reload_successful` metric after config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4260).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): fix bug with duplicated labels during stream aggregation via single-node VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4277).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmagent/) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).
diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go
index fe64ad03e..b606c477b 100644
--- a/lib/httpserver/httpserver.go
+++ b/lib/httpserver/httpserver.go
@@ -114,12 +114,6 @@ func serve(addr string, useProxyProtocol bool, rh RequestHandler, idx int) {
 	if tlsEnable.GetOptionalArg(idx) {
 		scheme = "https"
 	}
-	hostAddr := addr
-	if strings.HasPrefix(hostAddr, ":") {
-		hostAddr = "127.0.0.1" + hostAddr
-	}
-	logger.Infof("starting server at %s://%s/", scheme, hostAddr)
-	logger.Infof("pprof handlers are exposed at %s://%s/debug/pprof/", scheme, hostAddr)
 	var tlsConfig *tls.Config
 	if tlsEnable.GetOptionalArg(idx) {
 		certFile := tlsCertFile.GetOptionalArg(idx)
@@ -134,6 +128,9 @@ func serve(addr string, useProxyProtocol bool, rh RequestHandler, idx int) {
 	ln, err := netutil.NewTCPListener(scheme, addr, useProxyProtocol, tlsConfig)
 	if err != nil {
 		logger.Fatalf("cannot start http server at %s: %s", addr, err)
+	} else {
+		logger.Infof("started server at %s://%s/", scheme, ln.Addr())
+		logger.Infof("pprof handlers are exposed at %s://%s/debug/pprof/", scheme, ln.Addr())
 	}
 	serveWithListener(addr, ln, rh)
 }


Test patch:
diff --git a/apptest/README.md b/apptest/README.md
new file mode 100644
index 000000000..347bc5b1c
--- /dev/null
+++ b/apptest/README.md
@@ -0,0 +1,40 @@
+# App Integration Tests
+
+The `apptest` package contains the integration tests for the VictoriaMetrics
+applications (such as vmstorage, vminsert, and vmselect).
+
+An integration test aims at verifying the behavior of an application as a whole,
+as apposed to a unit test that verifies the behavior of a building block of an
+application.
+
+To achieve that an integration test starts an application in a separate process
+and then issues HTTP requets to it and verifies the responses, examines the
+metrics the app exposes and/or files it creates, etc.
+
+Note that an object of testing may be not just a single app, but several apps
+working together. A good example is VictoriaMetrics cluster. An integration test
+may reproduce an arbitrary cluster configuration and verify how the components
+work together as a system.
+
+The package provides a collection of helpers to start applications and make
+queries to them:
+
+-   `app.go` - contains the generic code for staring an application and should
+    not be used by integration tests directly.
+-   `{vmstorage,vminsert,etc}.go` - build on top of `app.go` and provide the
+    code for staring a specific application.
+-   `client.go` - provides helper functions for sending HTTP requests to
+    applications.
+
+The integration tests themselves reside in `*_test.go` files. Apart from having
+the `_test` suffix, there are no strict rules of how to name a file, but the
+name should reflect the prevailing purpose of the tests located in that file.
+For example, `sharding_test.go` aims at testing data sharding.
+
+Since integration tests start applications in a separate process, they require
+the application binary files to be built and put into the `bin` directory. The
+build rule used for running integration tests, `make integration-test`,
+accounts for that, it builds all application binaries before running the tests.
+But if you want to run the tests without `make`, i.e. by executing
+`go test ./app/apptest`, you will need to build the binaries first (for example,
+by executing `make all`).
diff --git a/apptest/app.go b/apptest/app.go
new file mode 100644
index 000000000..812861c8b
--- /dev/null
+++ b/apptest/app.go
@@ -0,0 +1,249 @@
+package apptest
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"reflect"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Regular expressions for runtime information to extract from the app logs.
+var (
+	storageDataPathRE = regexp.MustCompile(`successfully opened storage "(.*)"`)
+	httpListenAddrRE  = regexp.MustCompile(`started server at http://(.*:\d{1,5})/`)
+	vminsertAddrRE    = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
+	vmselectAddrRE    = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
+)
+
+// app represents an instance of some VictoriaMetrics server (such as vmstorage,
+// vminsert, or vmselect).
+type app struct {
+	instance string
+	binary   string
+	flags    []string
+	process  *os.Process
+}
+
+// appOptions holds the optional configuration of an app, such as default flags
+// to set and things to extract from the app's log.
+type appOptions struct {
+	defaultFlags map[string]string
+	extractREs   []*regexp.Regexp
+}
+
+// startApp starts an instance of an app using the app binary file path and
+// flags. When the opts are set, it also sets the default flag values and
+// extracts runtime information from the app's log.
+//
+// If the app has started successfully and all the requested items has been
+// extracted from logs, the function returns the instance of the app and the
+// extracted items. The extracted items are returned in the same order as the
+// corresponding extract regular expression have been provided in the opts.
+//
+// The function returns an error if the application has failed to start or the
+// function has timed out extracting items from the log (normally because no log
+// records match the regular expression).
+func startApp(instance string, binary string, flags []string, opts *appOptions) (*app, []string, error) {
+	flags = setDefaultFlags(flags, opts.defaultFlags)
+
+	cmd := exec.Command(binary, flags...)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, nil, err
+	}
+
+	app := &app{
+		instance: instance,
+		binary:   binary,
+		flags:    flags,
+		process:  cmd.Process,
+	}
+
+	go app.processOutput("stdout", stdout, app.writeToStderr)
+
+	lineProcessors := make([]lineProcessor, len(opts.extractREs))
+	reExtractors := make([]*reExtractor, len(opts.extractREs))
+	timeout := time.NewTimer(5 * time.Second).C
+	for i, re := range opts.extractREs {
+		reExtractors[i] = newREExtractor(re, timeout)
+		lineProcessors[i] = reExtractors[i].extractRE
+	}
+	go app.processOutput("stderr", stderr, append(lineProcessors, app.writeToStderr)...)
+
+	extracts, err := extractREs(reExtractors, timeout)
+	if err != nil {
+		app.Stop()
+		return nil, nil, err
+	}
+
+	return app, extracts, nil
+}
+
+// setDefaultFlags adds flags with default values to `flags` if it does not
+// initially contain them.
+func setDefaultFlags(flags []string, defaultFlags map[string]string) []string {
+	for _, flag := range flags {
+		for name := range defaultFlags {
+			if strings.HasPrefix(flag, name) {
+				delete(defaultFlags, name)
+				continue
+			}
+		}
+	}
+	for name, value := range defaultFlags {
+		flags = append(flags, name+"="+value)
+	}
+	return flags
+}
+
+// stop sends the app process a SIGINT signal and waits until it terminates
+// gracefully.
+func (app *app) Stop() {
+	if err := app.process.Signal(os.Interrupt); err != nil {
+		log.Fatalf("Could not send SIGINT signal to %s process: %v", app.instance, err)
+	}
+	if _, err := app.process.Wait(); err != nil {
+		log.Fatalf("Could not wait for %s process completion: %v", app.instance, err)
+	}
+}
+
+// String returns the string representation of the app state.
+func (app *app) String() string {
+	return fmt.Sprintf("{instance: %q binary: %q flags: %q}", app.instance, app.binary, app.flags)
+}
+
+// lineProcessor is a function that is applied to the each line of the app
+// output (stdout or stderr). The function returns true to indicate the caller
+// that it has completed its work and should not be called again.
+type lineProcessor func(line string) (done bool)
+
+// processOutput invokes a set of processors on each line of app output (stdout
+// or stderr). Once a line processor is done (returns true) it is never invoked
+// again.
+//
+// A simple use case for this is to pipe the output of the child process to the
+// output of the parent process. A more sophisticated one is to retrieve some
+// runtime information from the child process logs, such as the server's
+// host:port.
+func (app *app) processOutput(outputName string, output io.Reader, lps ...lineProcessor) {
+	activeLPs := map[int]lineProcessor{}
+	for i, lp := range lps {
+		activeLPs[i] = lp
+	}
+
+	scanner := bufio.NewScanner(output)
+	for scanner.Scan() {
+		line := scanner.Text()
+		for i, process := range activeLPs {
+			if process(line) {
+				delete(activeLPs, i)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Printf("could not scan %s %s: %v", app.instance, outputName, err)
+	}
+}
+
+// writeToStderr is a line processor that writes the line to the stderr.
+// The function always returns false to indicate its caller that each line must
+// be written to the stderr.
+func (app *app) writeToStderr(line string) bool {
+	fmt.Fprintf(os.Stderr, "%s %s\n", app.instance, line)
+	return false
+}
+
+// extractREs waits until all reExtractors return the result and then returns
+// the combined result with items ordered the same way as reExtractors.
+//
+// The function returns an error if timeout occurs sooner then all reExtractors
+// finish its work.
+func extractREs(reExtractors []*reExtractor, timeout <-chan time.Time) ([]string, error) {
+	n := len(reExtractors)
+	notFoundREs := make(map[int]string)
+	extracts := make([]string, n)
+	cases := make([]reflect.SelectCase, n+1)
+	for i, x := range reExtractors {
+		cases[i] = x.selectCase
+		notFoundREs[i] = x.re.String()
+	}
+	cases[n] = reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(timeout),
+	}
+
+	for notFound := n; notFound > 0; {
+		i, value, _ := reflect.Select(cases)
+		if i == n {
+			// n-th select case means timeout.
+
+			values := func(m map[int]string) []string {
+				s := []string{}
+				for _, v := range m {
+					s = append(s, v)
+				}
+				return s
+			}
+			return nil, fmt.Errorf("could not extract some or all regexps from stderr: %q", values(notFoundREs))
+		}
+		extracts[i] = value.String()
+		delete(notFoundREs, i)
+		notFound--
+	}
+	return extracts, nil
+}
+
+// reExtractor extracts some information based on a regular expression from the
+// app output within a timeout.
+type reExtractor struct {
+	re         *regexp.Regexp
+	result     chan string
+	timeout    <-chan time.Time
+	selectCase reflect.SelectCase
+}
+
+// newREExtractor create a new reExtractor based on a regexp and a timeout.
+func newREExtractor(re *regexp.Regexp, timeout <-chan time.Time) *reExtractor {
+	result := make(chan string)
+	return &reExtractor{
+		re:      re,
+		result:  result,
+		timeout: timeout,
+		selectCase: reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(result),
+		},
+	}
+}
+
+// extractRE is a line processor that extracts some information from a line
+// based on a regular expression. The function returns trun (to request the
+// caller to not to be called again) either when the match is found or due to
+// the timeout. The found match is written to the x.result channel and it is
+// important that this channel is monitored by a separate goroutine, otherwise
+// the function will block.
+func (x *reExtractor) extractRE(line string) bool {
+	submatch := x.re.FindSubmatch([]byte(line))
+	if len(submatch) == 2 {
+		select {
+		case x.result <- string(submatch[1]):
+		case <-x.timeout:
+		}
+		return true
+	}
+	return false
+}
diff --git a/apptest/client.go b/apptest/client.go
new file mode 100644
index 000000000..65ac4678f
--- /dev/null
+++ b/apptest/client.go
@@ -0,0 +1,130 @@
+package apptest
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Client is used for interacting with the apps over the network.
+//
+// At the moment it only supports HTTP protocol but may be exptended to support
+// RPCs, etc.
+type Client struct {
+	httpCli *http.Client
+}
+
+// NewClient creates a new client.
+func NewClient() *Client {
+	return &Client{
+		httpCli: &http.Client{
+			Transport: &http.Transport{},
+		},
+	}
+}
+
+// CloseConnections closes client connections.
+func (c *Client) CloseConnections() {
+	c.httpCli.CloseIdleConnections()
+}
+
+// Get sends a HTTP GET request. Once the function receives a response, it
+// checks whether the response status code matches the expected one and returns
+// the response body to the caller.
+func (c *Client) Get(t *testing.T, url string, wantStatusCode int) string {
+	t.Helper()
+	return c.do(t, http.MethodGet, url, "", "", wantStatusCode)
+}
+
+// Post sends a HTTP POST request. Once the function receives a response, it
+// checks whether the response status code matches the expected one and returns
+// the response body to the caller.
+func (c *Client) Post(t *testing.T, url, contentType, data string, wantStatusCode int) string {
+	t.Helper()
+	return c.do(t, http.MethodPost, url, contentType, data, wantStatusCode)
+}
+
+// PostForm sends a HTTP POST request containing the POST-form data. Once the
+// function receives a response, it checks whether the response status code
+// matches the expected one and returns the response body to the caller.
+func (c *Client) PostForm(t *testing.T, url string, data url.Values, wantStatusCode int) string {
+	t.Helper()
+	return c.Post(t, url, "application/x-www-form-urlencoded", data.Encode(), wantStatusCode)
+}
+
+// do prepares a HTTP request, sends it to the server, receives the response
+// from the server, ensures then response code matches the expected one, reads
+// the rentire response body and returns it to the caller.
+func (c *Client) do(t *testing.T, method, url, contentType, data string, wantStatusCode int) string {
+	t.Helper()
+
+	req, err := http.NewRequest(method, url, strings.NewReader(data))
+	if err != nil {
+		t.Fatalf("could not create a HTTP request: %v", err)
+	}
+
+	if len(contentType) > 0 {
+		req.Header.Add("Content-Type", contentType)
+	}
+	res, err := c.httpCli.Do(req)
+	if err != nil {
+		t.Fatalf("could not send HTTP request: %v", err)
+	}
+
+	body := readAllAndClose(t, res.Body)
+
+	if got, want := res.StatusCode, wantStatusCode; got != want {
+		t.Fatalf("unexpected response code: got %d, want %d (body: %s)", got, want, body)
+	}
+
+	return body
+}
+
+// readAllAndClose reads everything from the response body and then closes it.
+func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
+	t.Helper()
+
+	defer responseBody.Close()
+	b, err := io.ReadAll(responseBody)
+	if err != nil {
+		t.Fatalf("could not read response body: %d", err)
+	}
+	return string(b)
+}
+
+// ServesMetrics is used to retrive the app's metrics.
+//
+// This type is expected to be embdded by the apps that serve metrics.
+type ServesMetrics struct {
+	metricsURL string
+	cli        *Client
+}
+
+// GetIntMetric retrieves the value of a metric served by an app at /metrics URL.
+// The value is then converted to int.
+func (app *ServesMetrics) GetIntMetric(t *testing.T, metricName string) int {
+	return int(app.GetMetric(t, metricName))
+}
+
+// GetMetric retrieves the value of a metric served by an app at /metrics URL.
+func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
+	t.Helper()
+
+	metrics := app.cli.Get(t, app.metricsURL, http.StatusOK)
+	for _, metric := range strings.Split(metrics, "\n") {
+		value, found := strings.CutPrefix(metric, metricName)
+		if found {
+			value = strings.Trim(value, " ")
+			res, err := strconv.ParseFloat(value, 64)
+			if err != nil {
+				t.Fatalf("could not parse metric value %s: %v", metric, err)
+			}
+			return res
+		}
+	}
+	t.Fatalf("metic not found: %s", metricName)
+	return 0
+}
diff --git a/apptest/testcase.go b/apptest/testcase.go
new file mode 100644
index 000000000..6ec8455fe
--- /dev/null
+++ b/apptest/testcase.go
@@ -0,0 +1,42 @@
+package apptest
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+)
+
+// TestCase holds the state and defines clean-up procedure common for all test
+// cases.
+type TestCase struct {
+	t   *testing.T
+	cli *Client
+}
+
+// NewTestCase creates a new test case.
+func NewTestCase(t *testing.T) *TestCase {
+	return &TestCase{t, NewClient()}
+}
+
+// Dir returns the directory name that should be used by as the -storageDataDir.
+func (tc *TestCase) Dir() string {
+	return tc.t.Name()
+}
+
+// Client returns an instance of the client that can be used for interacting with
+// the app(s) under test.
+func (tc *TestCase) Client() *Client {
+	return tc.cli
+}
+
+// Close performs the test case clean up, such as closing all client connections
+// and removing the -storageDataDir directory.
+//
+// Note that the -storageDataDir is not removed in case of test case failure to
+// allow for furher manual debugging.
+func (tc *TestCase) Close() {
+	tc.cli.CloseConnections()
+	if !tc.t.Failed() {
+		fs.MustRemoveAll(tc.Dir())
+	}
+}
diff --git a/apptest/tests/multilevel_test.go b/apptest/tests/multilevel_test.go
new file mode 100644
index 000000000..ed1d0faf8
--- /dev/null
+++ b/apptest/tests/multilevel_test.go
@@ -0,0 +1,68 @@
+package tests
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestMultilevelSelect(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Close()
+
+	// Set up the following multi-level cluster configuration:
+	//
+	// vmselect (L2) -> vmselect (L1) -> vmstorage <- vminsert
+	//
+	// vmisert writes data into vmstorage.
+	// vmselect (L2) reads that data via vmselect (L1).
+
+	cli := tc.Client()
+
+	vmstorage := apptest.MustStartVmstorage(t, "vmstorage", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage",
+	}, cli)
+	defer vmstorage.Stop()
+	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
+		"-storageNode=" + vmstorage.VminsertAddr(),
+	}, cli)
+	defer vminsert.Stop()
+	vmselectL1 := apptest.MustStartVmselect(t, "vmselect-level1", []string{
+		"-storageNode=" + vmstorage.VmselectAddr(),
+	}, cli)
+	defer vmselectL1.Stop()
+	vmselectL2 := apptest.MustStartVmselect(t, "vmselect-level2", []string{
+		"-storageNode=" + vmselectL1.ClusternativeListenAddr(),
+	}, cli)
+	defer vmselectL2.Stop()
+
+	// Insert 1000 unique time series.Wait for 2 seconds to let vmstorage
+	// flush pending items so they become searchable.
+
+	const numMetrics = 1000
+	records := make([]string, numMetrics)
+	for i := range numMetrics {
+		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
+	}
+	vminsert.PrometheusAPIV1ImportPrometheus(t, "0", records)
+	time.Sleep(2 * time.Second)
+
+	// Retrieve all time series and verify that vmselect (L1) serves the complete
+	// set of time series.
+
+	seriesL1 := vmselectL1.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	if got, want := len(seriesL1.Data), numMetrics; got != want {
+		t.Fatalf("unexpected level-1 series count: got %d, want %d", got, want)
+	}
+
+	// Retrieve all time series and verify that vmselect (L2) serves the complete
+	// set of time series.
+
+	seriesL2 := vmselectL2.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	if got, want := len(seriesL2.Data), numMetrics; got != want {
+		t.Fatalf("unexpected level-2 series count: got %d, want %d", got, want)
+	}
+}
diff --git a/apptest/tests/sharding_test.go b/apptest/tests/sharding_test.go
new file mode 100644
index 000000000..dce0cfaa3
--- /dev/null
+++ b/apptest/tests/sharding_test.go
@@ -0,0 +1,84 @@
+package tests
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Close()
+
+	// Set up the following cluster configuration:
+	//
+	// - two vmstorage instances
+	// - vminsert points to the two vmstorages, its replication setting
+	//   is off which means it will only shard the incoming data across the two
+	//   vmstorages.
+	// - vmselect points to the two vmstorages and is expected to query both
+	//   vmstorages and build the full result out of the two partial results.
+
+	cli := tc.Client()
+
+	vmstorage1 := apptest.MustStartVmstorage(t, "vmstorage-1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
+	}, cli)
+	defer vmstorage1.Stop()
+	vmstorage2 := apptest.MustStartVmstorage(t, "vmstorage-2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
+	}, cli)
+	defer vmstorage2.Stop()
+	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	}, cli)
+	defer vminsert.Stop()
+	vmselect := apptest.MustStartVmselect(t, "vmselect", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	}, cli)
+	defer vmselect.Stop()
+
+	// Insert 1000 unique time series and verify the that inserted data has been
+	// indeed sharded by checking various metrics exposed by vminsert and
+	// vmstorage.
+	// Also wait for 2 seconds to let vminsert and vmstorage servers to update
+	// the values of the metrics they expose and to let vmstorages flush pending
+	// items so they become searchable.
+
+	const numMetrics = 1000
+	records := make([]string, numMetrics)
+	for i := range numMetrics {
+		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
+	}
+	vminsert.PrometheusAPIV1ImportPrometheus(t, "0", records)
+	time.Sleep(2 * time.Second)
+
+	numMetrics1 := vmstorage1.GetIntMetric(t, "vm_vminsert_metrics_read_total")
+	if numMetrics1 == 0 {
+		t.Fatalf("storage-1 has no time series")
+	}
+	numMetrics2 := vmstorage2.GetIntMetric(t, "vm_vminsert_metrics_read_total")
+	if numMetrics2 == 0 {
+		t.Fatalf("storage-2 has no time series")
+	}
+	if numMetrics1+numMetrics2 != numMetrics {
+		t.Fatalf("unxepected total number of metrics: vmstorage-1 (%d) + vmstorage-2 (%d) != %d", numMetrics1, numMetrics2, numMetrics)
+	}
+
+	// Retrieve all time series and verify that vmselect serves the complete set
+	//of time series.
+
+	series := vmselect.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	if got, want := series.Status, "success"; got != want {
+		t.Fatalf("unexpected /ap1/v1/series response status: got %s, want %s", got, want)
+	}
+	if got, want := series.IsPartial, false; got != want {
+		t.Fatalf("unexpected /ap1/v1/series response isPartial value: got %t, want %t", got, want)
+	}
+	if got, want := len(series.Data), numMetrics; got != want {
+		t.Fatalf("unexpected /ap1/v1/series response series count: got %d, want %d", got, want)
+	}
+}
diff --git a/apptest/vminsert.go b/apptest/vminsert.go
new file mode 100644
index 000000000..a4b723e9f
--- /dev/null
+++ b/apptest/vminsert.go
@@ -0,0 +1,77 @@
+package apptest
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// Vminsert holds the state of a vminsert app and provides vminsert-specific
+// functions.
+type Vminsert struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr string
+	cli            *Client
+}
+
+// MustStartVminsert is a test helper function that starts an instance of
+// vminsert and fails the test if the app fails to start.
+func MustStartVminsert(t *testing.T, instance string, flags []string, cli *Client) *Vminsert {
+	t.Helper()
+
+	app, err := StartVminsert(instance, flags, cli)
+	if err != nil {
+		t.Fatalf("Could not start %s: %v", instance, err)
+	}
+
+	return app
+}
+
+// StartVminsert starts an instance of vminsert with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVminsert(instance string, flags []string, cli *Client) (*Vminsert, error) {
+	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			httpListenAddrRE,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vminsert{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr: stderrExtracts[0],
+		cli:            cli,
+	}, nil
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to
+// /prometheus/api/v1/import/prometheus vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
+func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, tenant string, records []string) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, tenant)
+	app.cli.Post(t, url, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
+}
+
+// String returns the string representation of the vminsert app state.
+func (app *Vminsert) String() string {
+	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
+}
diff --git a/apptest/vmselect.go b/apptest/vmselect.go
new file mode 100644
index 000000000..69f300e49
--- /dev/null
+++ b/apptest/vmselect.go
@@ -0,0 +1,101 @@
+package apptest
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"testing"
+)
+
+// Vmselect holds the state of a vmselect app and provides vmselect-specific
+// functions.
+type Vmselect struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr          string
+	clusternativeListenAddr string
+	cli                     *Client
+}
+
+// MustStartVmselect is a test helper function that starts an instance of
+// vmselect and fails the test if the app fails to start.
+func MustStartVmselect(t *testing.T, instance string, flags []string, cli *Client) *Vmselect {
+	t.Helper()
+
+	app, err := StartVmselect(instance, flags, cli)
+	if err != nil {
+		t.Fatalf("Could not start %s: %v", instance, err)
+	}
+
+	return app
+}
+
+// StartVmselect starts an instance of vmselect with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmselect(instance string, flags []string, cli *Client) (*Vmselect, error) {
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr":          "127.0.0.1:0",
+			"-clusternativeListenAddr": "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			httpListenAddrRE,
+			vmselectAddrRE,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmselect{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr:          stderrExtracts[0],
+		clusternativeListenAddr: stderrExtracts[1],
+		cli:                     cli,
+	}, nil
+}
+
+// ClusternativeListenAddr returns the address at which the vmselect process is
+// listening for connections from other vmselect apps.
+func (app *Vmselect) ClusternativeListenAddr() string {
+	return app.clusternativeListenAddr
+}
+
+// PrometheusAPIV1SeriesResponse is an inmemory representation of the
+// /prometheus/api/v1/series response.
+type PrometheusAPIV1SeriesResponse struct {
+	Status    string
+	IsPartial bool
+	Data      []map[string]string
+}
+
+// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// and returns the list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1series
+func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, tenant, matchQuery string) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, tenant)
+	values := url.Values{}
+	values.Add("match[]", matchQuery)
+	jsonRes := app.cli.PostForm(t, seriesURL, values, http.StatusOK)
+	var res PrometheusAPIV1SeriesResponse
+	if err := json.Unmarshal([]byte(jsonRes), &res); err != nil {
+		t.Fatalf("could not unmarshal /api/v1/series response: %v", err)
+	}
+	return &res
+}
+
+// String returns the string representation of the vmselect app state.
+func (app *Vmselect) String() string {
+	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
+}
diff --git a/apptest/vmstorage.go b/apptest/vmstorage.go
new file mode 100644
index 000000000..db9633fd5
--- /dev/null
+++ b/apptest/vmstorage.go
@@ -0,0 +1,87 @@
+package apptest
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"time"
+)
+
+// Vmstorage holds the state of a vmstorage app and provides vmstorage-specific
+// functions.
+type Vmstorage struct {
+	*app
+	*ServesMetrics
+
+	storageDataPath string
+	httpListenAddr  string
+	vminsertAddr    string
+	vmselectAddr    string
+}
+
+// MustStartVmstorage is a test helper function that starts an instance of
+// vmstorage and fails the test if the app fails to start.
+func MustStartVmstorage(t *testing.T, instance string, flags []string, cli *Client) *Vmstorage {
+	t.Helper()
+
+	app, err := StartVmstorage(instance, flags, cli)
+	if err != nil {
+		t.Fatalf("Could not start %s: %v", instance, err)
+	}
+
+	return app
+}
+
+// StartVmstorage starts an instance of vmstorage with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmstorage(instance string, flags []string, cli *Client) (*Vmstorage, error) {
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmstorage", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
+			"-httpListenAddr":  "127.0.0.1:0",
+			"-vminsertAddr":    "127.0.0.1:0",
+			"-vmselectAddr":    "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			storageDataPathRE,
+			httpListenAddrRE,
+			vminsertAddrRE,
+			vmselectAddrRE,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmstorage{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
+			cli:        cli,
+		},
+		storageDataPath: stderrExtracts[0],
+		httpListenAddr:  stderrExtracts[1],
+		vminsertAddr:    stderrExtracts[2],
+		vmselectAddr:    stderrExtracts[3],
+	}, nil
+}
+
+// VminsertAddr returns the address at which the vmstorage process is listening
+// for vminsert connections.
+func (app *Vmstorage) VminsertAddr() string {
+	return app.vminsertAddr
+}
+
+// VmselectAddr returns the address at which the vmstorage process is listening
+// for vmselect connections.
+func (app *Vmstorage) VmselectAddr() string {
+	return app.vmselectAddr
+}
+
+// String returns the string representation of the vmstorage app state.
+func (app *Vmstorage) String() string {
+	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{
+		app.app, app.storageDataPath, app.httpListenAddr, app.vminsertAddr, app.vmselectAddr}...)
+}