Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1187 (2021-04-05 15:49:39)

deployment: add vmalert+alertmanager services and list of default alerts for cluster version


Patch:
diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json
index 6ccf17481..5dc53d346 100644
--- a/dashboards/victoriametrics.json
+++ b/dashboards/victoriametrics.json
@@ -5,7 +5,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "7.1.1"
+      "version": "7.5.1"
     },
     {
       "type": "panel",
@@ -50,7 +50,7 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1616012729496,
+  "iteration": 1617619884528,
   "links": [
     {
       "icon": "doc",
@@ -104,9 +104,7 @@
       "datasource": "$ds",
       "description": "How many datapoints are in storage",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "short",
@@ -196,9 +194,7 @@
       "datasource": "$ds",
       "description": "The minimum free disk space left among all storage nodes.",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "bytes",
@@ -288,9 +284,7 @@
       "datasource": "$ds",
       "description": "Total number of allocated CPUs for all VM components. ",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "short",
@@ -380,9 +374,7 @@
       "datasource": "$ds",
       "description": "Total size of allocated memory for all VM components.",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "bytes",
@@ -472,9 +464,7 @@
       "datasource": "$ds",
       "description": "How many entries inverted index contains. This value is proportional to the number of unique timeseries in storage(cardinality).",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "short",
@@ -564,9 +554,7 @@
       "datasource": "$ds",
       "description": "Total amount of used disk space",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "bytes",
@@ -657,9 +645,7 @@
       "decimals": 2,
       "description": "Average disk usage per datapoint.",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "bytes",
@@ -749,9 +735,7 @@
       "datasource": "$ds",
       "description": "Total size of allowed memory via flag `-memory.allowedPercent` for all VM components",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "format": "bytes",
@@ -833,14 +817,12 @@
       "columns": [],
       "datasource": "$ds",
       "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
+        "defaults": {},
         "overrides": []
       },
       "fontSize": "100%",
       "gridPos": {
-        "h": 7,
+        "h": 5,
         "w": 8,
         "x": 0,
         "y": 5
@@ -933,7 +915,6 @@
       "datasource": "$ds",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -941,7 +922,7 @@
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
-        "h": 7,
+        "h": 5,
         "w": 16,
         "x": 8,
         "y": 5
@@ -967,8 +948,11 @@
       "lines": true,
       "linewidth": 1,
       "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1037,7 +1021,6 @@
       "description": "How many datapoints are inserted into storage per second by protocol and accountID",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1048,7 +1031,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 12
+        "y": 10
       },
       "hiddenSeries": false,
       "id": 2,
@@ -1065,8 +1048,11 @@
       "lines": true,
       "linewidth": 1,
       "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1131,7 +1117,6 @@
       "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1142,7 +1127,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 12
+        "y": 10
       },
       "hiddenSeries": false,
       "id": 6,
@@ -1162,8 +1147,11 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1230,7 +1218,6 @@
       "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1241,7 +1228,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 20
+        "y": 18
       },
       "hiddenSeries": false,
       "id": 52,
@@ -1261,8 +1248,11 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1329,7 +1319,6 @@
       "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1340,7 +1329,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 20
+        "y": 18
       },
       "hiddenSeries": false,
       "id": 8,
@@ -1360,8 +1349,11 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1428,7 +1420,6 @@
       "description": "RPC errors are interconnection errors between cluster components. Errors rate should be 0 if network connection is stable and all components are up and operational.",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1439,7 +1430,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 28
+        "y": 26
       },
       "hiddenSeries": false,
       "id": 44,
@@ -1457,8 +1448,11 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 1,
       "points": false,
       "renderer": "flot",
@@ -1543,10 +1537,9 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:",
+      "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1557,31 +1550,29 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 28
+        "y": 26
       },
       "hiddenSeries": false,
-      "id": 12,
+      "id": 104,
       "legend": {
-        "avg": false,
-        "current": false,
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
         "max": false,
         "min": false,
         "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
-      "links": [
-        {
-          "targetBlank": true,
-          "title": "troubleshooting",
-          "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#troubleshooting"
-        }
-      ],
+      "links": [],
       "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1591,10 +1582,11 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance.*\", type=\"storage/hour_metric_ids\"})",
+          "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance.*\"}[5m])) by (job, level) ",
           "format": "time_series",
+          "hide": false,
           "intervalFactor": 1,
-          "legendFormat": "Active time series",
+          "legendFormat": "{{job}} - {{level}}",
           "refId": "A"
         }
       ],
@@ -1602,10 +1594,10 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Active time series ($instance)",
+      "title": "Logging rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1618,6 +1610,7 @@
       },
       "yaxes": [
         {
+          "decimals": null,
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -1630,7 +1623,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": "0",
+          "min": null,
           "show": true
         }
       ],
@@ -1645,10 +1638,9 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "Shows amount of remaining disk space at `-storageDataPath`",
+      "description": "Shows the percentage of remaining disk space at `-storageDataPath` by instance",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1659,7 +1651,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 36
+        "y": 34
       },
       "hiddenSeries": false,
       "id": 110,
@@ -1676,8 +1668,11 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1687,7 +1682,8 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(vm_free_disk_space_bytes{job=\"$job_storage\", instance=~\"$instance\"}) by(instance)",
+          "exemplar": true,
+          "expr": "sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\"}) by(instance) /\n(\n  sum(vm_free_disk_space_bytes{job=\"$job_storage\", instance=~\"$instance\"}) by(instance) +\n  sum(vm_data_size_bytes{job=\"$job_storage\", instance=~\"$instance\"}) by(instance)\n)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -1695,11 +1691,20 @@
           "refId": "A"
         }
       ],
-      "thresholds": [],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 0.8,
+          "yaxis": "left"
+        }
+      ],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Free disk space ($instance)",
+      "title": "Disk space used ($instance)",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1715,7 +1720,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "percentunit",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1742,10 +1747,9 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+      "description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": []
         },
         "overrides": []
@@ -1756,25 +1760,34 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 36
+        "y": 34
       },
       "hiddenSeries": false,
-      "id": 104,
+      "id": 12,
       "legend": {
         "avg": false,
         "current": false,
         "max": false,
         "min": false,
-        "show": true,
+        "show": false,
         "total": false,
         "values": false
       },
       "lines": true,
       "linewidth": 1,
-      "links": [],
+      "links": [
+        {
+          "targetBlank": true,
+          "title": "troubleshooting",
+          "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#troubleshooting"
+        }
+      ],
       "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
       "percentage": false,
-      "pluginVersion": "7.1.1",
+      "pluginVersion": "7.5.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1784,11 +1797,10 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance.*\"}[5m])) by (job, level) ",
+          "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance.*\", type=\"storage/hour_metric_ids\"})",
           "format": "time_series",
-          "hide": false,
           "intervalFactor": 1,
-          "legendFormat": "{{job}} - {{level}}",
+          "legendFormat": "Active time series",
           "refId": "A"
         }
       ],
@@ -1796,7 +1808,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Logging rate",
+      "title": "Active time series ($instance)",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1812,7 +1824,6 @@
       },
       "yaxes": [
         {
-          "decimals": null,
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -1825,7 +1836,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         }
       ],
@@ -1841,7 +1852,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 44
+        "y": 42
       },
       "id": 46,
       "panels": [
@@ -1854,7 +1865,6 @@
           "description": "Amount of used memory (resident)",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -1865,7 +1875,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 2
+            "y": 43
           },
           "hiddenSeries": false,
           "id": 66,
@@ -1882,8 +1892,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1905,7 +1918,7 @@
           "title": "Memory usage ($instance)",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -1947,7 +1960,6 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -1958,7 +1970,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 2
+            "y": 43
           },
           "hiddenSeries": false,
           "id": 64,
@@ -1975,8 +1987,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2043,7 +2058,6 @@
           "description": "Panel shows the number of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2054,7 +2068,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 10
+            "y": 51
           },
           "hiddenSeries": false,
           "id": 117,
@@ -2071,8 +2085,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2153,7 +2170,6 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2164,7 +2180,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 10
+            "y": 51
           },
           "hiddenSeries": false,
           "id": 72,
@@ -2181,8 +2197,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2248,7 +2267,6 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2259,7 +2277,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 18
+            "y": 59
           },
           "hiddenSeries": false,
           "id": 68,
@@ -2276,8 +2294,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2301,7 +2322,7 @@
           "title": "Goroutines ($instance)",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2341,11 +2362,9 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${ds}",
+          "datasource": "$ds",
           "fieldConfig": {
-            "defaults": {
-              "custom": {}
-            },
+            "defaults": {},
             "overrides": []
           },
           "fill": 1,
@@ -2354,7 +2373,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 18
+            "y": 59
           },
           "hiddenSeries": false,
           "id": 119,
@@ -2370,8 +2389,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2436,7 +2458,6 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2447,7 +2468,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 26
+            "y": 67
           },
           "hiddenSeries": false,
           "id": 70,
@@ -2464,8 +2485,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2489,7 +2513,7 @@
           "title": "Threads ($instance)",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2531,9 +2555,7 @@
           "dashes": false,
           "datasource": "$ds",
           "fieldConfig": {
-            "defaults": {
-              "custom": {}
-            },
+            "defaults": {},
             "overrides": []
           },
           "fill": 1,
@@ -2542,7 +2564,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 26
+            "y": 67
           },
           "hiddenSeries": false,
           "id": 120,
@@ -2558,8 +2580,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2627,7 +2652,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 45
+        "y": 43
       },
       "id": 106,
       "panels": [
@@ -2637,10 +2662,9 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "Shows how many of new time-series are created every second.  High churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
+          "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2651,7 +2675,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 3
+            "y": 44
           },
           "hiddenSeries": false,
           "id": 102,
@@ -2660,27 +2684,45 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "new series over 24h",
+              "yaxis": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
+              "exemplar": true,
               "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[5m]))",
+              "interval": "",
               "legendFormat": "churn rate",
               "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "new series over 24h",
+              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -2733,7 +2775,6 @@
           "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2744,7 +2785,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 3
+            "y": 44
           },
           "hiddenSeries": false,
           "id": 107,
@@ -2760,8 +2801,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2827,7 +2871,6 @@
           "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>50%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2838,7 +2881,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 52
           },
           "hiddenSeries": false,
           "id": 108,
@@ -2854,8 +2897,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2921,7 +2967,6 @@
           "description": "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n\nThis prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload.\n\nWhen limit is exceeded (graph is > 0) - extra labels are dropped, which could result in unexpected identical time series.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -2932,7 +2977,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 52
           },
           "hiddenSeries": false,
           "id": 116,
@@ -2949,8 +2994,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2960,6 +3008,7 @@
           "steppedLine": false,
           "targets": [
             {
+              "exemplar": true,
               "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=\"$job_insert\", instance=~\"$instance\"}[5m]))",
               "format": "time_series",
               "hide": false,
@@ -3022,7 +3071,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 46
+        "y": 44
       },
       "id": 48,
       "panels": [
@@ -3035,7 +3084,6 @@
           "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3046,7 +3094,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 77
+            "y": 45
           },
           "hiddenSeries": false,
           "id": 76,
@@ -3064,8 +3112,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
@@ -3139,7 +3190,6 @@
           "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` -  the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3150,7 +3200,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 77
+            "y": 45
           },
           "hiddenSeries": false,
           "id": 86,
@@ -3168,8 +3218,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
@@ -3257,7 +3310,6 @@
           "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3268,7 +3320,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 86
+            "y": 54
           },
           "hiddenSeries": false,
           "id": 80,
@@ -3286,8 +3338,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
@@ -3297,8 +3352,10 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(addr) > 0",
+              "exemplar": true,
+              "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(addr)",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{addr}}",
               "refId": "A"
@@ -3354,7 +3411,6 @@
           "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3365,7 +3421,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 86
+            "y": 54
           },
           "hiddenSeries": false,
           "id": 78,
@@ -3383,8 +3439,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
@@ -3394,8 +3453,10 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(addr) > 0",
+              "exemplar": true,
+              "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by(addr)",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{addr}}",
               "refId": "A"
@@ -3451,7 +3512,6 @@
           "description": "The number of rows or bytes that vminesrt internal buffer contains at the moment.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3462,7 +3522,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 94
+            "y": 62
           },
           "hiddenSeries": false,
           "id": 82,
@@ -3479,8 +3539,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3555,7 +3618,6 @@
           "description": "The number of rows that were lost because of unhealthy vmstorage nodes.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3566,7 +3628,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 94
+            "y": 62
           },
           "hiddenSeries": false,
           "id": 84,
@@ -3576,15 +3638,18 @@
             "current": true,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": true
           },
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3649,7 +3714,6 @@
           "description": "Network usage by internal VictoriaMetrics RPC protocol",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3660,7 +3724,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 102
+            "y": 70
           },
           "hiddenSeries": false,
           "id": 74,
@@ -3669,15 +3733,18 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3744,7 +3811,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 47
+        "y": 45
       },
       "id": 60,
       "panels": [
@@ -3757,7 +3824,6 @@
           "description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3768,7 +3834,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 54
+            "y": 46
           },
           "hiddenSeries": false,
           "id": 57,
@@ -3786,8 +3852,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3870,18 +3939,17 @@
           "description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
           },
-          "fill": 1,
+          "fill": 0,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 54
+            "y": 46
           },
           "hiddenSeries": false,
           "id": 58,
@@ -3898,8 +3966,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3909,9 +3980,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "1 - (sum(rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by (type) / sum(rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by (type))",
+              "exemplar": true,
+              "expr": "1 - (\n sum(rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by (type) /\n sum(rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[5m])) by (type)\n)",
               "format": "time_series",
               "hide": false,
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{type}}",
               "refId": "A"
@@ -3969,7 +4042,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 48
+        "y": 46
       },
       "id": 24,
       "panels": [
@@ -3982,7 +4055,6 @@
           "description": "Shows how many rows per second every storage node accepts. This metric doesn't show all stored rows since some of them may be dropped because of wrong timestamps or decode errors.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -3993,7 +4065,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 57
+            "y": 47
           },
           "hiddenSeries": false,
           "id": 100,
@@ -4002,7 +4074,7 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -4010,8 +4082,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4078,7 +4153,6 @@
           "description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4089,7 +4163,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 57
+            "y": 47
           },
           "hiddenSeries": false,
           "id": 113,
@@ -4098,7 +4172,7 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -4106,8 +4180,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4175,7 +4252,6 @@
           "description": "Shows how many datapoints are in the storage.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4186,7 +4262,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 65
+            "y": 55
           },
           "hiddenSeries": false,
           "id": 16,
@@ -4195,7 +4271,7 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -4203,8 +4279,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4273,7 +4352,6 @@
           "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second. The index datapoints value in general is much lower.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4284,7 +4362,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 65
+            "y": 55
           },
           "hiddenSeries": false,
           "id": 14,
@@ -4303,8 +4381,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4386,7 +4467,6 @@
           "description": "Shows amount of on-disk space occupied by data points.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4397,7 +4477,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 73
+            "y": 63
           },
           "hiddenSeries": false,
           "id": 18,
@@ -4406,7 +4486,7 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -4414,8 +4494,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4482,7 +4565,6 @@
           "description": "Shows amount of on-disk space occupied by inverted index.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4493,7 +4575,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 73
+            "y": 63
           },
           "hiddenSeries": false,
           "id": 20,
@@ -4502,7 +4584,7 @@
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -4510,8 +4592,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4578,7 +4663,6 @@
           "description": "The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4589,7 +4673,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 81
+            "y": 71
           },
           "hiddenSeries": false,
           "id": 54,
@@ -4605,8 +4689,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4672,7 +4759,6 @@
           "description": "The number of rows merged per second by storage nodes.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4683,7 +4769,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 81
+            "y": 71
           },
           "hiddenSeries": false,
           "id": 55,
@@ -4699,8 +4785,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4766,7 +4855,6 @@
           "description": "Shows the number of bytes read/write from the storage layer.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4777,7 +4865,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 89
+            "y": 79
           },
           "hiddenSeries": false,
           "id": 122,
@@ -4794,8 +4882,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4810,6 +4901,7 @@
           "steppedLine": false,
           "targets": [
             {
+              "exemplar": true,
               "expr": "sum(rate(process_io_storage_read_bytes_total{job=\"$job_storage\", instance=~\"$instance\"}[5m])) by(instance)",
               "format": "time_series",
               "hide": false,
@@ -4879,7 +4971,6 @@
           "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` -  small parts gradually merged into big parts (cold data)",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -4890,7 +4981,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 89
+            "y": 79
           },
           "hiddenSeries": false,
           "id": 22,
@@ -4907,8 +4998,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -4966,6 +5060,209 @@
             "align": false,
             "alignLevel": null
           }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 87
+          },
+          "hiddenSeries": false,
+          "id": 135,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.5.1",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(vm_rows_ignored_total{job=\"$job_storage\", instance=~\"$instance\"}) by (reason)",
+              "interval": "",
+              "legendFormat": "{{reason}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Rows ignored ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "fill": 0,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 87
+          },
+          "hiddenSeries": false,
+          "id": 133,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.5.1",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "max",
+              "color": "#C4162A"
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "max(vm_concurrent_addrows_capacity{job=\"$job_storage\", instance=~\"$instance\"})",
+              "interval": "",
+              "legendFormat": "max",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "sum(avg_over_time(vm_concurrent_addrows_current{job=\"$job_storage\", instance=~\"$instance\"}[1m])) by(instance)",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "{{instance}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Concurrent flushes on disk ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "title": "storage ($instance)",
@@ -4978,7 +5275,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 49
+        "y": 47
       },
       "id": 42,
       "panels": [
@@ -4991,7 +5288,6 @@
           "description": "Request rate accepted by vmselect nodes",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5022,8 +5318,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5090,7 +5389,6 @@
           "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 2 by default. May be configured with `search.maxConcurrentRequests` flag\n* `current` - current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means vmselect node is overloaded and require more CPU or higher limits.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5123,8 +5421,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5207,7 +5508,6 @@
           "description": "",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5238,8 +5538,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5293,7 +5596,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -5301,7 +5604,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             }
           ],
@@ -5321,7 +5624,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 50
+        "y": 48
       },
       "id": 40,
       "panels": [
@@ -5334,7 +5637,6 @@
           "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5345,7 +5647,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 49
           },
           "hiddenSeries": false,
           "id": 97,
@@ -5365,8 +5667,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5433,7 +5738,6 @@
           "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 4 by default. May be configured with `maxConcurrentInserts` flag\n* `current` - current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and require more CPU or higher limits.",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5444,7 +5748,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 49
           },
           "hiddenSeries": false,
           "id": 99,
@@ -5466,8 +5770,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5550,7 +5857,6 @@
           "description": "",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5561,7 +5867,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 57
           },
           "hiddenSeries": false,
           "id": 90,
@@ -5581,8 +5887,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5662,7 +5971,6 @@
           "description": "",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5673,7 +5981,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 57
           },
           "hiddenSeries": false,
           "id": 88,
@@ -5693,8 +6001,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5754,135 +6065,6 @@
             "alignLevel": null
           }
         },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": "$ds",
-          "description": "Shows the number of insert requests processed concurrently and concurrency limit.\n\n* `max` - controller by `-maxConcurrentInserts`\n* `current` - current number of goroutines busy with processing insert requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\n",
-          "fieldConfig": {
-            "defaults": {
-              "custom": {},
-              "links": []
-            },
-            "overrides": []
-          },
-          "fill": 1,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 0,
-            "y": 75
-          },
-          "hiddenSeries": false,
-          "id": 4,
-          "legend": {
-            "alignAsTable": true,
-            "avg": true,
-            "current": true,
-            "hideEmpty": false,
-            "hideZero": false,
-            "max": false,
-            "min": false,
-            "show": false,
-            "sort": "current",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [
-            {
-              "targetBlank": true,
-              "title": "Github discussion #1",
-              "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632#issuecomment-659717922"
-            },
-            {
-              "targetBlank": true,
-              "title": "Github discussion #2",
-              "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/618#issuecomment-658048293"
-            }
-          ],
-          "nullPointMode": "null",
-          "percentage": false,
-          "pluginVersion": "7.1.1",
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "max",
-              "color": "#C4162A",
-              "fill": 0
-            }
-          ],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(vm_concurrent_insert_capacity{job=\"$job_insert\", instance=~\"$instance\"})",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "max",
-              "refId": "A"
-            },
-            {
-              "expr": "sum(vm_concurrent_insert_current{job=\"$job_insert\", instance=~\"$instance\"})",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "current",
-              "refId": "B"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Concurrent inserts ($instance)",
-          "tooltip": {
-            "shared": true,
-            "sort": 2,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": 0,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            },
-            {
-              "decimals": 0,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
-        },
         {
           "aliasColors": {},
           "bars": false,
@@ -5892,7 +6074,6 @@
           "description": "Shows if vmstorage node is reachable for vminsert. If below 1 means vmstorage is not reachable at this moment of time. ",
           "fieldConfig": {
             "defaults": {
-              "custom": {},
               "links": []
             },
             "overrides": []
@@ -5903,7 +6084,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 75
+            "y": 65
           },
           "hiddenSeries": false,
           "id": 114,
@@ -5923,8 +6104,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.1",
+          "pluginVersion": "7.5.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -5989,7 +6173,7 @@
       "type": "row"
     }
   ],
-  "schemaVersion": 26,
+  "schemaVersion": 27,
   "style": "dark",
   "tags": [],
   "templating": {
@@ -6000,6 +6184,8 @@
           "text": "VictoriaMetrics",
           "value": "VictoriaMetrics"
         },
+        "description": null,
+        "error": null,
         "hide": 0,
         "includeAll": false,
         "label": null,
@@ -6017,13 +6203,18 @@
         "current": {},
         "datasource": "$ds",
         "definition": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
+        "description": null,
+        "error": null,
         "hide": 2,
         "includeAll": false,
         "label": null,
         "multi": false,
         "name": "job_insert",
         "options": [],
-        "query": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
+        "query": {
+          "query": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
+          "refId": "VictoriaMetrics-job_insert-Variable-Query"
+        },
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -6039,13 +6230,18 @@
         "current": {},
         "datasource": "$ds",
         "definition": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
+        "description": null,
+        "error": null,
         "hide": 2,
         "includeAll": false,
         "label": null,
         "multi": false,
         "name": "job_select",
         "options": [],
-        "query": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
+        "query": {
+          "query": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
+          "refId": "VictoriaMetrics-job_select-Variable-Query"
+        },
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -6061,13 +6257,18 @@
         "current": {},
         "datasource": "$ds",
         "definition": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
+        "description": null,
+        "error": null,
         "hide": 2,
         "includeAll": false,
         "label": null,
         "multi": false,
         "name": "job_storage",
         "options": [],
-        "query": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
+        "query": {
+          "query": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
+          "refId": "VictoriaMetrics-job_storage-Variable-Query"
+        },
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -6083,13 +6284,18 @@
         "current": {},
         "datasource": "$ds",
         "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+        "description": null,
+        "error": null,
         "hide": 0,
         "includeAll": true,
         "label": null,
         "multi": true,
         "name": "job",
         "options": [],
-        "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+        "query": {
+          "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+          "refId": "VictoriaMetrics-job-Variable-Query"
+        },
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -6105,23 +6311,24 @@
         "current": {},
         "datasource": "$ds",
         "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)",
+        "description": null,
+        "error": null,
         "hide": 0,
         "includeAll": true,
         "label": null,
         "multi": true,
         "name": "instance",
         "options": [],
-        "query": "label_values(vm_app_version{job=~\"$job\"}, instance)",
+        "query": {
+          "query": "label_values(vm_app_version{job=~\"$job\"}, instance)",
+          "refId": "VictoriaMetrics-instance-Variable-Query"
+        },
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,
         "tagValuesQuery": "label_values(vm_app_version{job=\"$tag\"}, instance)",
         "tags": [
-          {
-            "selected": false,
-            "text": "vmselect"
-          },
           {
             "selected": false,
             "text": "vmstorage"
@@ -6133,6 +6340,10 @@
           {
             "selected": false,
             "text": "vminsert"
+          },
+          {
+            "selected": false,
+            "text": "vmselect"
           }
         ],
         "tagsQuery": "label_values(vm_app_version, job)",
@@ -6142,7 +6353,7 @@
     ]
   },
   "time": {
-    "from": "now-30m",
+    "from": "now-3h",
     "to": "now"
   },
   "timepicker": {
diff --git a/deployment/docker/alertmanager.yml b/deployment/docker/alertmanager.yml
new file mode 100644
index 000000000..4b68f7863
--- /dev/null
+++ b/deployment/docker/alertmanager.yml
@@ -0,0 +1,5 @@
+route:
+  receiver: blackhole
+
+receivers:
+  - name: blackhole
\ No newline at end of file
diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml
new file mode 100644
index 000000000..fd61a943e
--- /dev/null
+++ b/deployment/docker/alerts.yml
@@ -0,0 +1,253 @@
+# File contains default list of alerts for VM cluster and vmagent services.
+# The alerts below are just recommendations and may require some updates
+# and threshold calibration according to every specific setup.
+groups:
+  - name: serviceHealth
+    rules:
+      # note the `job` filter and update accordingly to your setup
+      - alert: TooManyRestarts
+        expr: changes(process_start_time_seconds{job=~"vmselect|vminsert|vmstorage|vmagent|vmalert"}[15m]) > 2
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
+          description: "Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
+            It might be crashlooping."
+
+  # Alerts group for VM cluster assumes that Grafana dashboard
+  # https://grafana.com/grafana/dashboards/11176 is installed.
+  # Please, update the `dashboard` annotation according to your setup.
+  - name: vmcluster
+    interval: 30s
+    concurrency: 2
+    rules:
+      - alert: DiskRunsOutOfSpaceIn3Days
+        expr: |
+          vm_free_disk_space_bytes / ignoring(path) (
+           (
+            sum(rate(vm_rows_added_to_storage_total[1d])) -
+            sum(rate(vm_deduplicated_samples_total[1d])) without(type)
+           )
+          *
+           (
+            sum(vm_data_size_bytes{type!="indexdb"}) /
+            sum(vm_rows{type!="indexdb"})
+           )
+          ) < 3 * 24 * 3600
+        for: 30m
+        labels:
+          severity: critical
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=113&var-instance={{ $labels.instance }}"
+          summary: "Instance {{ $labels.instance }} will run out of disk space in 3 days"
+          description: "Taking into account current ingestion rate, free disk space will be enough only
+              for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.\n
+              Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible."
+
+      - alert: DiskRunsOutOfSpace
+        expr: |
+          sum(vm_data_size_bytes) by(instance) /
+          (
+           sum(vm_free_disk_space_bytes) by(instance) +
+           sum(vm_data_size_bytes) by(instance)
+          ) > 0.8
+        for: 30m
+        labels:
+          severity: critical
+        annotations:
+          dashboard: http://localhost:3000/d/oS7Bi_0Wz?viewPanel=110&var-instance={{ $labels.instance }}"
+          summary: "Instance {{ $labels.instance }} will run out of disk space soon"
+          description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
+            Having less than 20% of free disk space could cripple merges processes and overall performance.
+            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
+
+      - alert: RequestErrorsToAPI
+        expr: increase(vm_http_request_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for {{ $labels.job }} path {{ $labels.path }} (instance {{ $labels.instance }})"
+          description: "Requests to path {{ $labels.path }} are receiving errors.
+            Please verify if clients are sending correct requests."
+
+      - alert: RPCErrors
+        expr: sum(increase({__name__=~"vm_rpc_.*_errors_total"}[5m])) by(job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=44&var-instance={{ $labels.instance }}"
+          summary: "Too many RPC errors for {{ $labels.job }} (instance {{ $labels.instance }})"
+          description: "RPC errors are interconnection errors between cluster components.\n
+            Possible reasons for errors are misconfiguration, overload, network blips or unreachable components."
+
+      - alert: ConcurrentFlushesHitTheLimit
+        expr: avg_over_time(vm_concurrent_addrows_current[1m]) >= vm_concurrent_addrows_capacity
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=133&var-instance={{ $labels.instance }}"
+          summary: "vmstorage on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit"
+          description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n
+            When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU."
+
+      - alert: TooManyLogs
+        expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) by (job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=104&var-instance={{ $labels.instance }}"
+          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
+           Worth to check logs for specific error messages."
+
+      - alert: RowsRejectedOnIngestion
+        expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=135&var-instance={{ $labels.instance }}"
+          summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
+          description: "VM is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the
+            following reason: \"{{ $labels.reason }}\""
+
+      - alert: TooHighChurnRate
+        expr: |
+          (
+             sum(rate(vm_new_timeseries_created_total[5m]))
+             /
+             sum(rate(vm_rows_inserted_total[5m]))
+           ) > 0.1
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=102"
+          summary: "Churn rate is more than 10% for the last 15m"
+          description: "VM constantly creates new time series.\n
+            This effect is known as Churn Rate.\n
+            High Churn Rate tightly connected with database performance and may
+            result in unexpected OOM's or slow queries."
+
+      - alert: TooHighChurnRate24h
+        expr: |
+          sum(increase(vm_new_timeseries_created_total[24h]))
+          >
+          (sum(vm_cache_entries{type="storage/hour_metric_ids"})* 3)
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=102"
+          summary: "Too high number of new series created over last 24h"
+          description: "The number of created new time series over last 24h is 3x times higher than
+            current number of active series.\n
+            This effect is known as Churn Rate.\n
+            High Churn Rate tightly connected with database performance and may
+            result in unexpected OOM's or slow queries."
+
+      - alert: TooHighSlowInsertsRate
+        expr: |
+          (
+             sum(rate(vm_slow_row_inserts_total[5m]))
+             /
+             sum(rate(vm_rows_inserted_total[5m]))
+           ) > 0.05
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=108"
+          summary: "Percentage of slow inserts is more than 5% for the last 15m"
+          description: "High rate of slow inserts may be a sign of resource exhaustion
+            for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series."
+
+      - alert: ProcessNearFDLimits
+        expr: (process_max_fds - process_open_fds) < 100
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=117&var-instance={{ $labels.instance }}"
+          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
+          description: "Exhausting OS file descriptors limit can cause severe degradation of the process.
+          Consider to increase the limit as fast as possible."
+
+      - alert: LabelsLimitExceededOnIngestion
+        expr: sum(increase(vm_metrics_with_dropped_labels_total[5m])) by (instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=116&var-instance={{ $labels.instance }}"
+          summary: "Metrics ingested to vminsert on ({{ $labels.instance }}) are exceeding labels limit"
+          description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
+           This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
+           correctly or that clients which send these metrics aren't misbehaving."
+
+      - alert: VminsertIsDroppingRows
+        expr: sum(rate(vm_rpc_rows_lost_total[5m])) by(instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=84&var-instance={{ $labels.instance }}"
+          summary: "VMinsert \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops rows due to RPC errors."
+          description: "VMinsert starts to drop rows if there are no healthy VMstorage nodes where it can route
+           insert requests to. Check the health state of VMstorage nodes and RPC metrics."
+
+
+  # Alerts group for vmagent assumes that Grafana dashboard
+  # https://grafana.com/grafana/dashboards/12683 is installed.
+  # Pls update the `dashboard` annotation according to your setup.
+  - name: vmagent
+    interval: 30s
+    concurrency: 2
+    rules:
+      - alert: PersistentQueueIsDroppingData
+        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) by (job, instance) > 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=49&var-instance={{ $labels.instance }}"
+          summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
+          description: "Vmagent dropped {{ $value | humanize1024 }} from persistent queue
+              on instance {{ $labels.instance }} for the last 10m."
+
+      - alert: TooManyScrapeErrors
+        expr: sum(increase(vm_promscrape_scrapes_failed_total[5m])) by (job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=31&var-instance={{ $labels.instance }}"
+          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
+
+      - alert: TooManyWriteErrors
+        expr: |
+          (sum(increase(vm_ingestserver_request_errors_total[5m])) by (job, instance)
+          +
+          sum(increase(vmagent_http_request_errors_total[5m])) by (job, instance)) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance }}"
+          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
+
+      - alert: TooManyRemoteWriteErrors
+        expr: sum(rate(vmagent_remotewrite_retries_count_total[5m])) by(job, instance, url) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=61&var-instance={{ $labels.instance }}"
+          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
+          description: "Vmagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n
+            Ensure that destination is up and reachable."
diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
index c0e319499..28571c736 100644
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@@ -64,6 +64,38 @@ services:
       - 8481:8481
     restart: always
 
+  vmalert:
+    container_name: vmalert
+    image: victoriametrics/vmalert
+    depends_on:
+      - "vmselect"
+    ports:
+      - 8880:8880
+    volumes:
+      - ./alerts.yml:/etc/alerts/alerts.yml
+    command:
+      - '--datasource.url=http://vmselect:8481/select/0/prometheus'
+      - '--remoteRead.url=http://vmselect:8481/select/0/prometheus'
+      - '--remoteWrite.url=http://vminsert:8480/insert/0/prometheus'
+      - '--notifier.url=http://alertmanager:9093/'
+      - '--rule=/etc/alerts/*.yml'
+      # display source of alerts in grafana
+      - '-external.url=http://127.0.0.1:3000' #grafana outside container
+      # when copypaste the line below be aware of '$$' for escaping in '$expr'
+      - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":"{{$$expr|quotesEscape|crlfEscape|queryEscape}}"},{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
+    restart: always
+
+  alertmanager:
+    container_name: alertmanager
+    image:  prom/alertmanager
+    volumes:
+      - ./alertmanager.yml:/config/alertmanager.yml
+    command:
+      - '--config.file=/config/alertmanager.yml'
+    ports:
+      - 9093:9093
+    restart: always
+
 volumes:
   vmagentdata: {}
   strgdata: {}
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 75f1096bd..f1294214e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -14,6 +14,7 @@
 * FEATURE: vmagent: add AWS IAM roles for tasks support for EC2 service discovery according to [these docs](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
 * FEATURE: vmagent: add support for `proxy_tls_config`, `proxy_authorization`, `proxy_basic_auth`, `proxy_bearer_token` and `proxy_bearer_token_file` options in `consul_sd_config`, `dockerswarm_sd_config` and `eureka_sd_config` sections.
 * FEATURE: vmagent: pass `X-Prometheus-Scrape-Timeout-Seconds` header to scrape targets as Prometheus does. In this case scrape targets can limit the time needed for performing the scrape. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813118733) for details.
+* FEATURE: vmagent: drop corrupted persistent queue files at `-remoteWrite.tmpDataPath` instead of throwing a fatal error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1030).
 * FEATURE: vmauth: add support for authorization via [bearer token](https://swagger.io/docs/specification/authentication/bearer-authentication/). See [the docs](https://victoriametrics.github.io/vmauth.html#auth-config) for details.
 
 * BUGFIX: vmagent: properly work with simple HTTP proxies which don't support `CONNECT` method. For example, [PushProx](https://github.com/prometheus-community/PushProx). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179).
diff --git a/lib/persistentqueue/fastqueue.go b/lib/persistentqueue/fastqueue.go
index 5c25933f6..7dda73a86 100644
--- a/lib/persistentqueue/fastqueue.go
+++ b/lib/persistentqueue/fastqueue.go
@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// FastQueue is a wrapper around Queue, which prefers sending data via memory.
+// FastQueue is fast persistent queue, which prefers sending data via memory.
 //
 // It falls back to sending data via file when readers don't catch up with writers.
 type FastQueue struct {
@@ -20,7 +20,7 @@ type FastQueue struct {
 	cond sync.Cond
 
 	// pq is file-based queue
-	pq *Queue
+	pq *queue
 
 	// ch is in-memory queue
 	ch chan *bytesutil.ByteBuffer
@@ -40,7 +40,7 @@ type FastQueue struct {
 // Otherwise its size is limited by maxPendingBytes. The oldest data is dropped when the queue
 // reaches maxPendingSize.
 func MustOpenFastQueue(path, name string, maxInmemoryBlocks, maxPendingBytes int) *FastQueue {
-	pq := MustOpen(path, name, maxPendingBytes)
+	pq := mustOpen(path, name, maxPendingBytes)
 	fq := &FastQueue{
 		pq: pq,
 		ch: make(chan *bytesutil.ByteBuffer, maxInmemoryBlocks),
@@ -174,7 +174,12 @@ func (fq *FastQueue) MustReadBlock(dst []byte) ([]byte, bool) {
 			return dst, true
 		}
 		if n := fq.pq.GetPendingBytes(); n > 0 {
-			return fq.pq.MustReadBlock(dst)
+			data, ok := fq.pq.MustReadBlockNonblocking(dst)
+			if ok {
+				return data, true
+			}
+			dst = data
+			continue
 		}
 
 		// There are no blocks. Wait for new block.
diff --git a/lib/persistentqueue/persistentqueue.go b/lib/persistentqueue/persistentqueue.go
index 916276c3f..4c082a9f8 100644
--- a/lib/persistentqueue/persistentqueue.go
+++ b/lib/persistentqueue/persistentqueue.go
@@ -8,7 +8,6 @@ import (
 	"os"
 	"regexp"
 	"strconv"
-	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -26,8 +25,10 @@ const defaultChunkFileSize = (MaxBlockSize + 8) * 16
 
 var chunkFileNameRegex = regexp.MustCompile("^[0-9A-F]{16}$")
 
-// Queue represents persistent queue.
-type Queue struct {
+// queue represents persistent queue.
+//
+// It is unsafe to call queue methods from concurrent goroutines.
+type queue struct {
 	chunkFileSize   uint64
 	maxBlockSize    uint64
 	maxPendingBytes uint64
@@ -37,13 +38,6 @@ type Queue struct {
 
 	flockF *os.File
 
-	// mu protects all the fields below.
-	mu sync.Mutex
-
-	// cond is used for notifying blocked readers when new data has been added
-	// or when MustClose is called.
-	cond sync.Cond
-
 	reader            *filestream.Reader
 	readerPath        string
 	readerOffset      uint64
@@ -74,10 +68,7 @@ type Queue struct {
 // ResetIfEmpty resets q if it is empty.
 //
 // This is needed in order to remove chunk file associated with empty q.
-func (q *Queue) ResetIfEmpty() {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
+func (q *queue) ResetIfEmpty() {
 	if q.readerOffset != q.writerOffset {
 		// The queue isn't empty.
 		return
@@ -86,10 +77,13 @@ func (q *Queue) ResetIfEmpty() {
 		// The file is too small to drop. Leave it as is in order to reduce filesystem load.
 		return
 	}
+	q.mustResetFiles()
+}
+
+func (q *queue) mustResetFiles() {
 	if q.readerPath != q.writerPath {
 		logger.Panicf("BUG: readerPath=%q doesn't match writerPath=%q", q.readerPath, q.writerPath)
 	}
-
 	q.reader.MustClose()
 	q.writer.MustClose()
 	fs.MustRemoveAll(q.readerPath)
@@ -115,31 +109,29 @@ func (q *Queue) ResetIfEmpty() {
 	}
 	q.reader = r
 
-	if err := q.flushMetainfoLocked(); err != nil {
+	if err := q.flushMetainfo(); err != nil {
 		logger.Panicf("FATAL: cannot flush metainfo: %s", err)
 	}
 }
 
 // GetPendingBytes returns the number of pending bytes in the queue.
-func (q *Queue) GetPendingBytes() uint64 {
-	q.mu.Lock()
+func (q *queue) GetPendingBytes() uint64 {
 	n := q.writerOffset - q.readerOffset
-	q.mu.Unlock()
 	return n
 }
 
-// MustOpen opens persistent queue from the given path.
+// mustOpen opens persistent queue from the given path.
 //
 // If maxPendingBytes is greater than 0, then the max queue size is limited by this value.
 // The oldest data is deleted when queue size exceeds maxPendingBytes.
-func MustOpen(path, name string, maxPendingBytes int) *Queue {
+func mustOpen(path, name string, maxPendingBytes int) *queue {
 	if maxPendingBytes < 0 {
 		maxPendingBytes = 0
 	}
-	return mustOpen(path, name, defaultChunkFileSize, MaxBlockSize, uint64(maxPendingBytes))
+	return mustOpenInternal(path, name, defaultChunkFileSize, MaxBlockSize, uint64(maxPendingBytes))
 }
 
-func mustOpen(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) *Queue {
+func mustOpenInternal(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) *queue {
 	if chunkFileSize < 8 || chunkFileSize-8 < maxBlockSize {
 		logger.Panicf("BUG: too small chunkFileSize=%d for maxBlockSize=%d; chunkFileSize must fit at least one block", chunkFileSize, maxBlockSize)
 	}
@@ -166,15 +158,14 @@ func mustCreateFlockFile(path string) *os.File {
 	return f
 }
 
-func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) (*Queue, error) {
+func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) (*queue, error) {
 	// Protect from concurrent opens.
-	var q Queue
+	var q queue
 	q.chunkFileSize = chunkFileSize
 	q.maxBlockSize = maxBlockSize
 	q.maxPendingBytes = maxPendingBytes
 	q.dir = path
 	q.name = name
-	q.cond.L = &q.mu
 
 	q.blocksDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_dropped_total{path=%q}`, path))
 	q.bytesDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_dropped_total{path=%q}`, path))
@@ -346,17 +337,8 @@ func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingB
 
 // MustClose closes q.
 //
-// It unblocks all the MustReadBlock calls.
-//
 // MustWriteBlock mustn't be called during and after the call to MustClose.
-func (q *Queue) MustClose() {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
-	// Unblock goroutines blocked on cond in MustReadBlock.
-	q.mustStop = true
-	q.cond.Broadcast()
-
+func (q *queue) MustClose() {
 	// Close writer.
 	q.writer.MustClose()
 	q.writer = nil
@@ -366,7 +348,7 @@ func (q *Queue) MustClose() {
 	q.reader = nil
 
 	// Store metainfo
-	if err := q.flushMetainfoLocked(); err != nil {
+	if err := q.flushMetainfo(); err != nil {
 		logger.Panicf("FATAL: cannot flush chunked queue metainfo: %s", err)
 	}
 
@@ -377,11 +359,11 @@ func (q *Queue) MustClose() {
 	q.flockF = nil
 }
 
-func (q *Queue) chunkFilePath(offset uint64) string {
+func (q *queue) chunkFilePath(offset uint64) string {
 	return fmt.Sprintf("%s/%016X", q.dir, offset)
 }
 
-func (q *Queue) metainfoPath() string {
+func (q *queue) metainfoPath() string {
 	return q.dir + "/metainfo.json"
 }
 
@@ -390,14 +372,10 @@ func (q *Queue) metainfoPath() string {
 // The block size cannot exceed MaxBlockSize.
 //
 // It is safe calling this function from concurrent goroutines.
-func (q *Queue) MustWriteBlock(block []byte) {
+func (q *queue) MustWriteBlock(block []byte) {
 	if uint64(len(block)) > q.maxBlockSize {
 		logger.Panicf("BUG: too big block to send: %d bytes; it mustn't exceed %d bytes", len(block), q.maxBlockSize)
 	}
-
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
 	if q.mustStop {
 		logger.Panicf("BUG: MustWriteBlock cannot be called after MustClose")
 	}
@@ -416,7 +394,10 @@ func (q *Queue) MustWriteBlock(block []byte) {
 		bb := blockBufPool.Get()
 		for q.writerOffset-q.readerOffset > maxPendingBytes {
 			var err error
-			bb.B, err = q.readBlockLocked(bb.B[:0])
+			bb.B, err = q.readBlock(bb.B[:0])
+			if err == errEmptyQueue {
+				break
+			}
 			if err != nil {
 				logger.Panicf("FATAL: cannot read the oldest block %s", err)
 			}
@@ -429,38 +410,18 @@ func (q *Queue) MustWriteBlock(block []byte) {
 			return
 		}
 	}
-	if err := q.writeBlockLocked(block); err != nil {
+	if err := q.writeBlock(block); err != nil {
 		logger.Panicf("FATAL: %s", err)
 	}
-
-	// Notify blocked reader if any.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/484 for details.
-	q.cond.Signal()
 }
 
 var blockBufPool bytesutil.ByteBufferPool
 
-func (q *Queue) writeBlockLocked(block []byte) error {
+func (q *queue) writeBlock(block []byte) error {
 	if q.writerLocalOffset+q.maxBlockSize+8 > q.chunkFileSize {
-		// Finalize the current chunk and start new one.
-		q.writer.MustClose()
-		// There is no need to do fs.MustSyncPath(q.writerPath) here,
-		// since MustClose already does this.
-		if n := q.writerOffset % q.chunkFileSize; n > 0 {
-			q.writerOffset += (q.chunkFileSize - n)
+		if err := q.nextChunkFileForWrite(); err != nil {
+			return fmt.Errorf("cannot create next chunk file: %w", err)
 		}
-		q.writerFlushedOffset = q.writerOffset
-		q.writerLocalOffset = 0
-		q.writerPath = q.chunkFilePath(q.writerOffset)
-		w, err := filestream.Create(q.writerPath, false)
-		if err != nil {
-			return fmt.Errorf("cannot create chunk file %q: %w", q.writerPath, err)
-		}
-		q.writer = w
-		if err := q.flushMetainfoLocked(); err != nil {
-			return fmt.Errorf("cannot flush metainfo: %w", err)
-		}
-		fs.MustSyncPath(q.dir)
 	}
 
 	// Write block len.
@@ -479,62 +440,61 @@ func (q *Queue) writeBlockLocked(block []byte) error {
 	}
 	q.blocksWritten.Inc()
 	q.bytesWritten.Add(len(block))
-	return q.flushMetainfoIfNeededLocked(true)
+	return q.flushWriterMetainfoIfNeeded()
 }
 
-// MustReadBlock appends the next block from q to dst and returns the result.
-//
-// false is returned after MustClose call.
-//
-// It is safe calling this function from concurrent goroutines.
-func (q *Queue) MustReadBlock(dst []byte) ([]byte, bool) {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
-	for {
-		if q.mustStop {
-			return dst, false
-		}
-		if q.readerOffset > q.writerOffset {
-			logger.Panicf("BUG: readerOffset=%d cannot exceed writerOffset=%d", q.readerOffset, q.writerOffset)
-		}
-		if q.readerOffset < q.writerOffset {
-			break
-		}
-		q.cond.Wait()
+func (q *queue) nextChunkFileForWrite() error {
+	// Finalize the current chunk and start new one.
+	q.writer.MustClose()
+	// There is no need to do fs.MustSyncPath(q.writerPath) here,
+	// since MustClose already does this.
+	if n := q.writerOffset % q.chunkFileSize; n > 0 {
+		q.writerOffset += q.chunkFileSize - n
 	}
+	q.writerFlushedOffset = q.writerOffset
+	q.writerLocalOffset = 0
+	q.writerPath = q.chunkFilePath(q.writerOffset)
+	w, err := filestream.Create(q.writerPath, false)
+	if err != nil {
+		return fmt.Errorf("cannot create chunk file %q: %w", q.writerPath, err)
+	}
+	q.writer = w
+	if err := q.flushMetainfo(); err != nil {
+		return fmt.Errorf("cannot flush metainfo: %w", err)
+	}
+	fs.MustSyncPath(q.dir)
+	return nil
+}
 
-	data, err := q.readBlockLocked(dst)
+// MustReadBlockNonblocking appends the next block from q to dst and returns the result.
+//
+// false is returned if q is empty.
+func (q *queue) MustReadBlockNonblocking(dst []byte) ([]byte, bool) {
+	if q.readerOffset > q.writerOffset {
+		logger.Panicf("BUG: readerOffset=%d cannot exceed writerOffset=%d", q.readerOffset, q.writerOffset)
+	}
+	if q.readerOffset == q.writerOffset {
+		return dst, false
+	}
+	var err error
+	dst, err = q.readBlock(dst)
 	if err != nil {
-		// Skip the current chunk, since it may be broken.
-		q.readerOffset += q.chunkFileSize - q.readerOffset%q.chunkFileSize
-		_ = q.flushMetainfoLocked()
+		if err == errEmptyQueue {
+			return dst, false
+		}
 		logger.Panicf("FATAL: %s", err)
 	}
-	return data, true
+	return dst, true
 }
 
-func (q *Queue) readBlockLocked(dst []byte) ([]byte, error) {
+func (q *queue) readBlock(dst []byte) ([]byte, error) {
 	if q.readerLocalOffset+q.maxBlockSize+8 > q.chunkFileSize {
-		// Remove the current chunk and go to the next chunk.
-		q.reader.MustClose()
-		fs.MustRemoveAll(q.readerPath)
-		if n := q.readerOffset % q.chunkFileSize; n > 0 {
-			q.readerOffset += (q.chunkFileSize - n)
+		if err := q.nextChunkFileForRead(); err != nil {
+			return dst, fmt.Errorf("cannot open next chunk file: %w", err)
 		}
-		q.readerLocalOffset = 0
-		q.readerPath = q.chunkFilePath(q.readerOffset)
-		r, err := filestream.Open(q.readerPath, true)
-		if err != nil {
-			return dst, fmt.Errorf("cannot open chunk file %q: %w", q.readerPath, err)
-		}
-		q.reader = r
-		if err := q.flushMetainfoLocked(); err != nil {
-			return dst, fmt.Errorf("cannot flush metainfo: %w", err)
-		}
-		fs.MustSyncPath(q.dir)
 	}
 
+again:
 	// Read block len.
 	header := headerBufPool.Get()
 	header.B = bytesutil.Resize(header.B, 8)
@@ -542,27 +502,73 @@ func (q *Queue) readBlockLocked(dst []byte) ([]byte, error) {
 	blockLen := encoding.UnmarshalUint64(header.B)
 	headerBufPool.Put(header)
 	if err != nil {
-		return dst, fmt.Errorf("cannot read header with size 8 bytes from %q: %w", q.readerPath, err)
+		logger.Errorf("skipping corrupted %q, since header with size 8 bytes cannot be read from it: %s", q.readerPath, err)
+		if err := q.skipBrokenChunkFile(); err != nil {
+			return dst, err
+		}
+		goto again
 	}
 	if blockLen > q.maxBlockSize {
-		return dst, fmt.Errorf("too big block size read from %q: %d bytes; cannot exceed %d bytes", q.readerPath, blockLen, q.maxBlockSize)
+		logger.Errorf("skipping corrupted %q, since too big block size is read from it: %d bytes; cannot exceed %d bytes", q.readerPath, blockLen, q.maxBlockSize)
+		if err := q.skipBrokenChunkFile(); err != nil {
+			return dst, err
+		}
+		goto again
 	}
 
 	// Read block contents.
 	dstLen := len(dst)
 	dst = bytesutil.Resize(dst, dstLen+int(blockLen))
 	if err := q.readFull(dst[dstLen:]); err != nil {
-		return dst, fmt.Errorf("cannot read block contents with size %d bytes from %q: %w", blockLen, q.readerPath, err)
+		logger.Errorf("skipping corrupted %q, since contents with size %d bytes cannot be read from it: %s", q.readerPath, blockLen, err)
+		if err := q.skipBrokenChunkFile(); err != nil {
+			return dst[:dstLen], err
+		}
+		goto again
 	}
 	q.blocksRead.Inc()
 	q.bytesRead.Add(int(blockLen))
-	if err := q.flushMetainfoIfNeededLocked(false); err != nil {
+	if err := q.flushReaderMetainfoIfNeeded(); err != nil {
 		return dst, err
 	}
 	return dst, nil
 }
 
-func (q *Queue) write(buf []byte) error {
+func (q *queue) skipBrokenChunkFile() error {
+	// Try to recover from broken chunk file by skipping it.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1030
+	q.readerOffset += q.chunkFileSize - q.readerOffset%q.chunkFileSize
+	if q.readerOffset >= q.writerOffset {
+		q.mustResetFiles()
+		return errEmptyQueue
+	}
+	return q.nextChunkFileForRead()
+}
+
+var errEmptyQueue = fmt.Errorf("the queue is empty")
+
+func (q *queue) nextChunkFileForRead() error {
+	// Remove the current chunk and go to the next chunk.
+	q.reader.MustClose()
+	fs.MustRemoveAll(q.readerPath)
+	if n := q.readerOffset % q.chunkFileSize; n > 0 {
+		q.readerOffset += q.chunkFileSize - n
+	}
+	q.readerLocalOffset = 0
+	q.readerPath = q.chunkFilePath(q.readerOffset)
+	r, err := filestream.Open(q.readerPath, true)
+	if err != nil {
+		return fmt.Errorf("cannot open chunk file %q: %w", q.readerPath, err)
+	}
+	q.reader = r
+	if err := q.flushMetainfo(); err != nil {
+		return fmt.Errorf("cannot flush metainfo: %w", err)
+	}
+	fs.MustSyncPath(q.dir)
+	return nil
+}
+
+func (q *queue) write(buf []byte) error {
 	bufLen := uint64(len(buf))
 	n, err := q.writer.Write(buf)
 	if err != nil {
@@ -576,7 +582,7 @@ func (q *Queue) write(buf []byte) error {
 	return nil
 }
 
-func (q *Queue) readFull(buf []byte) error {
+func (q *queue) readFull(buf []byte) error {
 	bufLen := uint64(len(buf))
 	if q.readerOffset+bufLen > q.writerFlushedOffset {
 		q.writer.MustFlush(false)
@@ -594,22 +600,32 @@ func (q *Queue) readFull(buf []byte) error {
 	return nil
 }
 
-func (q *Queue) flushMetainfoIfNeededLocked(flushData bool) error {
+func (q *queue) flushReaderMetainfoIfNeeded() error {
 	t := fasttime.UnixTimestamp()
 	if t == q.lastMetainfoFlushTime {
 		return nil
 	}
-	if flushData {
-		q.writer.MustFlush(true)
+	if err := q.flushMetainfo(); err != nil {
+		return fmt.Errorf("cannot flush metainfo: %w", err)
+	}
+	q.lastMetainfoFlushTime = t
+	return nil
+}
+
+func (q *queue) flushWriterMetainfoIfNeeded() error {
+	t := fasttime.UnixTimestamp()
+	if t == q.lastMetainfoFlushTime {
+		return nil
 	}
-	if err := q.flushMetainfoLocked(); err != nil {
+	q.writer.MustFlush(true)
+	if err := q.flushMetainfo(); err != nil {
 		return fmt.Errorf("cannot flush metainfo: %w", err)
 	}
 	q.lastMetainfoFlushTime = t
 	return nil
 }
 
-func (q *Queue) flushMetainfoLocked() error {
+func (q *queue) flushMetainfo() error {
 	mi := &metainfo{
 		Name:         q.name,
 		ReaderOffset: q.readerOffset,
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index b4a8d5d2c..1f93c4be5 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -60,6 +60,15 @@ type Config struct {
 	baseDir string
 }
 
+func (cfg *Config) mustStart() {
+	startTime := time.Now()
+	logger.Infof("starting service discovery routines...")
+	for i := range cfg.ScrapeConfigs {
+		cfg.ScrapeConfigs[i].mustStart(cfg.baseDir)
+	}
+	logger.Infof("started service discovery routines in %.3f seconds", time.Since(startTime).Seconds())
+}
+
 func (cfg *Config) mustStop() {
 	startTime := time.Now()
 	logger.Infof("stopping service discovery routines...")
@@ -120,6 +129,21 @@ type ScrapeConfig struct {
 	swc *scrapeWorkConfig
 }
 
+func (sc *ScrapeConfig) mustStart(baseDir string) {
+	for i := range sc.KubernetesSDConfigs {
+		swosFunc := func(metaLabels map[string]string) interface{} {
+			target := metaLabels["__address__"]
+			sw, err := sc.swc.getScrapeWork(target, nil, metaLabels)
+			if err != nil {
+				logger.Errorf("cannot create kubernetes_sd_config target %q for job_name %q: %s", target, sc.swc.jobName, err)
+				return nil
+			}
+			return sw
+		}
+		sc.KubernetesSDConfigs[i].MustStart(baseDir, swosFunc)
+	}
+}
+
 func (sc *ScrapeConfig) mustStop() {
 	for i := range sc.KubernetesSDConfigs {
 		sc.KubernetesSDConfigs[i].MustStop()
@@ -243,15 +267,7 @@ func (cfg *Config) getKubernetesSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 		ok := true
 		for j := range sc.KubernetesSDConfigs {
 			sdc := &sc.KubernetesSDConfigs[j]
-			swos, err := sdc.GetScrapeWorkObjects(cfg.baseDir, func(metaLabels map[string]string) interface{} {
-				target := metaLabels["__address__"]
-				sw, err := sc.swc.getScrapeWork(target, nil, metaLabels)
-				if err != nil {
-					logger.Errorf("cannot create kubernetes_sd_config target %q for job_name %q: %s", target, sc.swc.jobName, err)
-					return nil
-				}
-				return sw
-			})
+			swos, err := sdc.GetScrapeWorkObjects()
 			if err != nil {
 				logger.Errorf("skipping kubernetes_sd_config targets for job_name %q because of error: %s", sc.swc.jobName, err)
 				ok = false
diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index a09221087..8afa6b613 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
 // apiConfig contains config for API server
@@ -15,18 +14,12 @@ type apiConfig struct {
 	aw *apiWatcher
 }
 
-func (ac *apiConfig) mustStop() {
-	ac.aw.mustStop()
+func (ac *apiConfig) mustStart() {
+	ac.aw.mustStart()
 }
 
-var configMap = discoveryutils.NewConfigMap()
-
-func getAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) {
-	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir, swcFunc) })
-	if err != nil {
-		return nil, err
-	}
-	return v.(*apiConfig), nil
+func (ac *apiConfig) mustStop() {
+	ac.aw.mustStop()
 }
 
 func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) {
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index eb27c1040..d5def2f29 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -72,6 +72,10 @@ func newAPIWatcher(apiServer string, ac *promauth.Config, sdc *SDConfig, swcFunc
 	}
 }
 
+func (aw *apiWatcher) mustStart() {
+	aw.gw.startWatchersForRole(aw.role, aw)
+}
+
 func (aw *apiWatcher) mustStop() {
 	aw.gw.unsubscribeAPIWatcher(aw)
 	aw.swosByNamespaceLock.Lock()
@@ -128,7 +132,8 @@ func (aw *apiWatcher) getScrapeWorkObjectsForLabels(labelss []map[string]string)
 
 // getScrapeWorkObjects returns all the ScrapeWork objects for the given aw.
 func (aw *apiWatcher) getScrapeWorkObjects() []interface{} {
-	aw.gw.startWatchersForRole(aw.role, aw)
+	aw.gw.registerPendingAPIWatchers()
+
 	aw.swosByNamespaceLock.Lock()
 	defer aw.swosByNamespaceLock.Unlock()
 
@@ -185,8 +190,12 @@ func newGroupWatcher(apiServer string, ac *promauth.Config, namespaces []string,
 }
 
 func getGroupWatcher(apiServer string, ac *promauth.Config, namespaces []string, selectors []Selector, proxyURL *url.URL) *groupWatcher {
-	key := fmt.Sprintf("apiServer=%s, namespaces=%s, selectors=%s, proxyURL=%v, authConfig=%s",
-		apiServer, namespaces, selectorsKey(selectors), proxyURL, ac.String())
+	proxyURLStr := "<nil>"
+	if proxyURL != nil {
+		proxyURLStr = proxyURL.String()
+	}
+	key := fmt.Sprintf("apiServer=%s, namespaces=%s, selectors=%s, proxyURL=%s, authConfig=%s",
+		apiServer, namespaces, selectorsKey(selectors), proxyURLStr, ac.String())
 	groupWatchersLock.Lock()
 	gw := groupWatchers[key]
 	if gw == nil {
@@ -223,12 +232,57 @@ func (gw *groupWatcher) getObjectByRole(role, namespace, name string) object {
 		// this is needed for testing
 		return nil
 	}
+	o := gw.getCachedObjectByRole(role, namespace, name)
+	if o != nil {
+		// Fast path: the object has been found in the cache.
+		return o
+	}
+
+	// The object wasn't found in the cache. Try querying it directly from API server.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1182#issuecomment-813353359 for details.
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_loads_total{role=%q}`, role)).Inc()
+	objectType := getObjectTypeByRole(role)
+	path := getAPIPath(objectType, namespace, "")
+	path += "/" + name
+	requestURL := gw.apiServer + path
+	resp, err := gw.doRequest(requestURL)
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_load_errors_total{role=%q}`, role)).Inc()
+		logger.Errorf("cannot obtain data for object %s (namespace=%q, name=%q): %s", role, namespace, name, err)
+		return nil
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_load_errors_total{role=%q}`, role)).Inc()
+		logger.Errorf("cannot read response from %q: %s", requestURL, err)
+		return nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_load_misses_total{role=%q}`, role)).Inc()
+			return nil
+		}
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_load_errors_total{role=%q}`, role)).Inc()
+		logger.Errorf("unexpected status code when reading response from %q; got %d; want %d; response body: %q", requestURL, resp.StatusCode, http.StatusOK, data)
+		return nil
+	}
+	parseObject, _ := getObjectParsersForRole(role)
+	o, err = parseObject(data)
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_direct_object_load_errors_total{role=%q}`, role)).Inc()
+		logger.Errorf("cannot parse object obtained from %q: %s; response body: %q", requestURL, err, data)
+		return nil
+	}
+	// There is no need in storing the object in urlWatcher cache, since it should be eventually populated there by urlWatcher itself.
+	return o
+}
+
+func (gw *groupWatcher) getCachedObjectByRole(role, namespace, name string) object {
 	key := namespace + "/" + name
 	gw.startWatchersForRole(role, nil)
-	gw.mu.Lock()
-	defer gw.mu.Unlock()
-
-	for _, uw := range gw.m {
+	uws := gw.getURLWatchers()
+	for _, uw := range uws {
 		if uw.role != role {
 			// Role mismatch
 			continue
@@ -263,7 +317,9 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 			uw.reloadObjects()
 			go uw.watchForUpdates()
 		}
-		uw.subscribeAPIWatcher(aw)
+		if aw != nil {
+			uw.subscribeAPIWatcher(aw)
+		}
 	}
 }
 
@@ -279,12 +335,28 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
 	return gw.client.Do(req)
 }
 
-func (gw *groupWatcher) unsubscribeAPIWatcher(aw *apiWatcher) {
+func (gw *groupWatcher) registerPendingAPIWatchers() {
+	uws := gw.getURLWatchers()
+	for _, uw := range uws {
+		uw.registerPendingAPIWatchers()
+	}
+}
+
+func (gw *groupWatcher) getURLWatchers() []*urlWatcher {
 	gw.mu.Lock()
+	uws := make([]*urlWatcher, 0, len(gw.m))
 	for _, uw := range gw.m {
-		uw.unsubscribeAPIWatcher(aw)
+		uws = append(uws, uw)
 	}
 	gw.mu.Unlock()
+	return uws
+}
+
+func (gw *groupWatcher) unsubscribeAPIWatcher(aw *apiWatcher) {
+	uws := gw.getURLWatchers()
+	for _, uw := range uws {
+		uw.unsubscribeAPIWatcher(aw)
+	}
 }
 
 // urlWatcher watches for an apiURL and updates object states in objectsByKey.
@@ -297,9 +369,16 @@ type urlWatcher struct {
 	parseObject     parseObjectFunc
 	parseObjectList parseObjectListFunc
 
-	// mu protects aws, objectsByKey and resourceVersion
+	// mu protects aws, awsPending, objectsByKey and resourceVersion
 	mu sync.Mutex
 
+	// awsPending contains pending apiWatcher objects, which are registered in a batch.
+	// Batch registering saves CPU time needed for registering big number of Kubernetes objects
+	// shared among big number of scrape jobs, since per-object labels are generated only once
+	// for all the scrape jobs (each scrape job is associated with a single apiWatcher).
+	// See reloadScrapeWorksForAPIWatchers for details.
+	awsPending map[*apiWatcher]struct{}
+
 	// aws contains registered apiWatcher objects
 	aws map[*apiWatcher]struct{}
 
@@ -327,6 +406,7 @@ func newURLWatcher(role, namespace, apiURL string, gw *groupWatcher) *urlWatcher
 		parseObject:     parseObject,
 		parseObjectList: parseObjectList,
 
+		awsPending:   make(map[*apiWatcher]struct{}),
 		aws:          make(map[*apiWatcher]struct{}),
 		objectsByKey: make(map[string]object),
 
@@ -341,30 +421,38 @@ func newURLWatcher(role, namespace, apiURL string, gw *groupWatcher) *urlWatcher
 }
 
 func (uw *urlWatcher) subscribeAPIWatcher(aw *apiWatcher) {
-	if aw == nil {
-		return
-	}
 	uw.mu.Lock()
 	if _, ok := uw.aws[aw]; !ok {
-		swosByKey := make(map[string][]interface{})
-		for key, o := range uw.objectsByKey {
-			labels := o.getTargetLabels(uw.gw)
-			swos := aw.getScrapeWorkObjectsForLabels(labels)
-			if len(swos) > 0 {
-				swosByKey[key] = swos
-			}
+		if _, ok := uw.awsPending[aw]; !ok {
+			uw.awsPending[aw] = struct{}{}
+			metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscribers{role=%q,status="pending"}`, uw.role)).Inc()
 		}
-		aw.reloadScrapeWorks(uw.namespace, swosByKey)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscibers{role=%q}`, uw.role)).Inc()
 	}
 	uw.mu.Unlock()
 }
 
+func (uw *urlWatcher) registerPendingAPIWatchers() {
+	uw.mu.Lock()
+	awsPending := make([]*apiWatcher, 0, len(uw.awsPending))
+	for aw := range uw.awsPending {
+		awsPending = append(awsPending, aw)
+		delete(uw.awsPending, aw)
+	}
+	uw.reloadScrapeWorksForAPIWatchers(awsPending, uw.objectsByKey)
+	uw.mu.Unlock()
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscribers{role=%q,status="working"}`, uw.role)).Add(len(awsPending))
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscribers{role=%q,status="pending"}`, uw.role)).Add(-len(awsPending))
+}
+
 func (uw *urlWatcher) unsubscribeAPIWatcher(aw *apiWatcher) {
 	uw.mu.Lock()
+	if _, ok := uw.awsPending[aw]; ok {
+		delete(uw.awsPending, aw)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscribers{role=%q,status="pending"}`, uw.role)).Dec()
+	}
 	if _, ok := uw.aws[aw]; ok {
 		delete(uw.aws, aw)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscibers{role=%q}`, uw.role)).Dec()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vm_promscrape_discovery_kubernetes_subscribers{role=%q,status="working"}`, uw.role)).Dec()
 	}
 	uw.mu.Unlock()
 }
@@ -625,32 +713,32 @@ func parseError(data []byte) (*Error, error) {
 }
 
 func getAPIPathsWithNamespaces(role string, namespaces []string, selectors []Selector) ([]string, []string) {
-	objectName := getObjectNameByRole(role)
-	if objectName == "nodes" || len(namespaces) == 0 {
+	objectType := getObjectTypeByRole(role)
+	if objectType == "nodes" || len(namespaces) == 0 {
 		query := joinSelectors(role, selectors)
-		path := getAPIPath(objectName, "", query)
+		path := getAPIPath(objectType, "", query)
 		return []string{path}, []string{""}
 	}
 	query := joinSelectors(role, selectors)
 	paths := make([]string, len(namespaces))
 	for i, namespace := range namespaces {
-		paths[i] = getAPIPath(objectName, namespace, query)
+		paths[i] = getAPIPath(objectType, namespace, query)
 	}
 	return paths, namespaces
 }
 
-func getAPIPath(objectName, namespace, query string) string {
-	suffix := objectName
+func getAPIPath(objectType, namespace, query string) string {
+	suffix := objectType
 	if namespace != "" {
-		suffix = "namespaces/" + namespace + "/" + objectName
+		suffix = "namespaces/" + namespace + "/" + objectType
 	}
 	if len(query) > 0 {
 		suffix += "?" + query
 	}
-	if objectName == "ingresses" {
+	if objectType == "ingresses" {
 		return "/apis/networking.k8s.io/v1beta1/" + suffix
 	}
-	if objectName == "endpointslices" {
+	if objectType == "endpointslices" {
 		return "/apis/discovery.k8s.io/v1beta1/" + suffix
 	}
 	return "/api/v1/" + suffix
@@ -679,7 +767,7 @@ func joinSelectors(role string, selectors []Selector) string {
 	return strings.Join(args, "&")
 }
 
-func getObjectNameByRole(role string) string {
+func getObjectTypeByRole(role string) string {
 	switch role {
 	case "node":
 		return "nodes"
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 805a88b01..af4b16b5e 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -139,8 +139,10 @@ func appendEndpointLabelsForAddresses(ms []map[string]string, gw *groupWatcher,
 	eas []EndpointAddress, epp EndpointPort, svc *Service, ready string) []map[string]string {
 	for _, ea := range eas {
 		var p *Pod
-		if o := gw.getObjectByRole("pod", ea.TargetRef.Namespace, ea.TargetRef.Name); o != nil {
-			p = o.(*Pod)
+		if ea.TargetRef.Name != "" {
+			if o := gw.getObjectByRole("pod", ea.TargetRef.Namespace, ea.TargetRef.Name); o != nil {
+				p = o.(*Pod)
+			}
 		}
 		m := getEndpointLabelsForAddressAndPort(podPortsSeen, eps, ea, epp, p, svc, ready)
 		ms = append(ms, m)
diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go
index 9315730f2..28160743b 100644
--- a/lib/promscrape/discovery/kubernetes/kubernetes.go
+++ b/lib/promscrape/discovery/kubernetes/kubernetes.go
@@ -17,6 +17,9 @@ type SDConfig struct {
 	ProxyURL         proxy.URL                 `yaml:"proxy_url,omitempty"`
 	Namespaces       Namespaces                `yaml:"namespaces,omitempty"`
 	Selectors        []Selector                `yaml:"selectors,omitempty"`
+
+	cfg      *apiConfig
+	startErr error
 }
 
 // Namespaces represents namespaces for SDConfig
@@ -37,23 +40,33 @@ type Selector struct {
 // ScrapeWorkConstructorFunc must construct ScrapeWork object for the given metaLabels.
 type ScrapeWorkConstructorFunc func(metaLabels map[string]string) interface{}
 
-// GetScrapeWorkObjects returns ScrapeWork objects for the given sdc and baseDir.
+// GetScrapeWorkObjects returns ScrapeWork objects for the given sdc.
+//
+// This function must be called after MustStart call.
+func (sdc *SDConfig) GetScrapeWorkObjects() ([]interface{}, error) {
+	if sdc.cfg == nil {
+		return nil, sdc.startErr
+	}
+	return sdc.cfg.aw.getScrapeWorkObjects(), nil
+}
+
+// MustStart initializes sdc before its usage.
 //
 // swcFunc is used for constructing such objects.
-func (sdc *SDConfig) GetScrapeWorkObjects(baseDir string, swcFunc ScrapeWorkConstructorFunc) ([]interface{}, error) {
-	cfg, err := getAPIConfig(sdc, baseDir, swcFunc)
+func (sdc *SDConfig) MustStart(baseDir string, swcFunc ScrapeWorkConstructorFunc) {
+	cfg, err := newAPIConfig(sdc, baseDir, swcFunc)
 	if err != nil {
-		return nil, fmt.Errorf("cannot create API config: %w", err)
+		sdc.startErr = fmt.Errorf("cannot create API config for kubernetes: %w", err)
+		return
 	}
-	return cfg.aw.getScrapeWorkObjects(), nil
+	cfg.aw.mustStart()
+	sdc.cfg = cfg
 }
 
 // MustStop stops further usage for sdc.
 func (sdc *SDConfig) MustStop() {
-	v := configMap.Delete(sdc)
-	if v != nil {
-		// v can be nil if GetLabels wasn't called yet.
-		cfg := v.(*apiConfig)
-		cfg.mustStop()
+	if sdc.cfg != nil {
+		// sdc.cfg can be nil on MustStart error.
+		sdc.cfg.mustStop()
 	}
 }
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 861c35c4d..7dd454891 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -93,6 +93,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	if err != nil {
 		logger.Fatalf("cannot read %q: %s", configFile, err)
 	}
+	cfg.mustStart()
 
 	scs := newScrapeConfigs(pushData)
 	scs.add("static_configs", 0, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })
@@ -130,6 +131,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 				goto waitForChans
 			}
 			cfg.mustStop()
+			cfgNew.mustStart()
 			cfg = cfgNew
 			data = dataNew
 		case <-tickerCh:
@@ -143,6 +145,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 				goto waitForChans
 			}
 			cfg.mustStop()
+			cfgNew.mustStart()
 			cfg = cfgNew
 			data = dataNew
 		case <-globalStopCh:


Test patch:
diff --git a/lib/persistentqueue/persistentqueue_test.go b/lib/persistentqueue/persistentqueue_test.go
index e4b83d540..e135eb415 100644
--- a/lib/persistentqueue/persistentqueue_test.go
+++ b/lib/persistentqueue/persistentqueue_test.go
@@ -5,16 +5,14 @@ import (
 	"io/ioutil"
 	"os"
 	"strconv"
-	"sync"
 	"testing"
-	"time"
 )
 
 func TestQueueOpenClose(t *testing.T) {
 	path := "queue-open-close"
 	mustDeleteDir(path)
 	for i := 0; i < 3; i++ {
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		if n := q.GetPendingBytes(); n > 0 {
 			t.Fatalf("pending bytes must be 0; got %d", n)
 		}
@@ -28,7 +26,7 @@ func TestQueueOpen(t *testing.T) {
 		path := "queue-open-invalid-metainfo"
 		mustCreateDir(path)
 		mustCreateFile(path+"/metainfo.json", "foobarbaz")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -38,7 +36,7 @@ func TestQueueOpen(t *testing.T) {
 		mustCreateEmptyMetainfo(path, "foobar")
 		mustCreateFile(path+"/junk-file", "foobar")
 		mustCreateDir(path + "/junk-dir")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -47,7 +45,7 @@ func TestQueueOpen(t *testing.T) {
 		mustCreateDir(path)
 		mustCreateEmptyMetainfo(path, "foobar")
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 1234), "qwere")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -56,7 +54,7 @@ func TestQueueOpen(t *testing.T) {
 		mustCreateDir(path)
 		mustCreateEmptyMetainfo(path, "foobar")
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 100*uint64(defaultChunkFileSize)), "asdf")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -72,7 +70,7 @@ func TestQueueOpen(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 0), "adfsfd")
-		q := MustOpen(path, mi.Name, 0)
+		q := mustOpen(path, mi.Name, 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -86,7 +84,7 @@ func TestQueueOpen(t *testing.T) {
 		if err := mi.WriteToFile(path + "/metainfo.json"); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		q := MustOpen(path, mi.Name, 0)
+		q := mustOpen(path, mi.Name, 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -94,7 +92,7 @@ func TestQueueOpen(t *testing.T) {
 		path := "queue-open-metainfo-dir"
 		mustCreateDir(path)
 		mustCreateDir(path + "/metainfo.json")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -110,7 +108,7 @@ func TestQueueOpen(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 0), "sdf")
-		q := MustOpen(path, mi.Name, 0)
+		q := mustOpen(path, mi.Name, 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -119,7 +117,7 @@ func TestQueueOpen(t *testing.T) {
 		mustCreateDir(path)
 		mustCreateEmptyMetainfo(path, "foobar")
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 0), "sdfdsf")
-		q := MustOpen(path, "foobar", 0)
+		q := mustOpen(path, "foobar", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -133,7 +131,7 @@ func TestQueueOpen(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		mustCreateFile(fmt.Sprintf("%s/%016X", path, 0), "sdf")
-		q := MustOpen(path, "baz", 0)
+		q := mustOpen(path, "baz", 0)
 		q.MustClose()
 		mustDeleteDir(path)
 	})
@@ -142,7 +140,7 @@ func TestQueueOpen(t *testing.T) {
 func TestQueueResetIfEmpty(t *testing.T) {
 	path := "queue-reset-if-empty"
 	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", 0)
+	q := mustOpen(path, "foobar", 0)
 	defer func() {
 		q.MustClose()
 		mustDeleteDir(path)
@@ -154,14 +152,18 @@ func TestQueueResetIfEmpty(t *testing.T) {
 		for i := 0; i < 10; i++ {
 			q.MustWriteBlock(block)
 			var ok bool
-			buf, ok = q.MustReadBlock(buf[:0])
+			buf, ok = q.MustReadBlockNonblocking(buf[:0])
 			if !ok {
-				t.Fatalf("unexpected ok=false returned from MustReadBlock")
+				t.Fatalf("unexpected ok=false returned from MustReadBlockNonblocking")
 			}
 		}
 		q.ResetIfEmpty()
 		if n := q.GetPendingBytes(); n > 0 {
-			t.Fatalf("unexpected non-zer pending bytes after queue reset: %d", n)
+			t.Fatalf("unexpected non-zero pending bytes after queue reset: %d", n)
+		}
+		q.ResetIfEmpty()
+		if n := q.GetPendingBytes(); n > 0 {
+			t.Fatalf("unexpected non-zero pending bytes after queue reset: %d", n)
 		}
 	}
 }
@@ -169,7 +171,7 @@ func TestQueueResetIfEmpty(t *testing.T) {
 func TestQueueWriteRead(t *testing.T) {
 	path := "queue-write-read"
 	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", 0)
+	q := mustOpen(path, "foobar", 0)
 	defer func() {
 		q.MustClose()
 		mustDeleteDir(path)
@@ -188,9 +190,9 @@ func TestQueueWriteRead(t *testing.T) {
 		var buf []byte
 		var ok bool
 		for _, block := range blocks {
-			buf, ok = q.MustReadBlock(buf[:0])
+			buf, ok = q.MustReadBlockNonblocking(buf[:0])
 			if !ok {
-				t.Fatalf("unexpected ok=%v returned from MustReadBlock; want true", ok)
+				t.Fatalf("unexpected ok=%v returned from MustReadBlockNonblocking; want true", ok)
 			}
 			if string(buf) != string(block) {
 				t.Fatalf("unexpected block read; got %q; want %q", buf, block)
@@ -205,7 +207,7 @@ func TestQueueWriteRead(t *testing.T) {
 func TestQueueWriteCloseRead(t *testing.T) {
 	path := "queue-write-close-read"
 	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", 0)
+	q := mustOpen(path, "foobar", 0)
 	defer func() {
 		q.MustClose()
 		mustDeleteDir(path)
@@ -222,16 +224,16 @@ func TestQueueWriteCloseRead(t *testing.T) {
 			t.Fatalf("pending bytes must be greater than 0; got %d", n)
 		}
 		q.MustClose()
-		q = MustOpen(path, "foobar", 0)
+		q = mustOpen(path, "foobar", 0)
 		if n := q.GetPendingBytes(); n <= 0 {
 			t.Fatalf("pending bytes must be greater than 0; got %d", n)
 		}
 		var buf []byte
 		var ok bool
 		for _, block := range blocks {
-			buf, ok = q.MustReadBlock(buf[:0])
+			buf, ok = q.MustReadBlockNonblocking(buf[:0])
 			if !ok {
-				t.Fatalf("unexpected ok=%v returned from MustReadBlock; want true", ok)
+				t.Fatalf("unexpected ok=%v returned from MustReadBlockNonblocking; want true", ok)
 			}
 			if string(buf) != string(block) {
 				t.Fatalf("unexpected block read; got %q; want %q", buf, block)
@@ -243,137 +245,12 @@ func TestQueueWriteCloseRead(t *testing.T) {
 	}
 }
 
-func TestQueueReadEmpty(t *testing.T) {
-	path := "queue-read-empty"
-	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", 0)
-	defer mustDeleteDir(path)
-
-	resultCh := make(chan error)
-	go func() {
-		data, ok := q.MustReadBlock(nil)
-		var err error
-		if ok {
-			err = fmt.Errorf("unexpected ok=%v returned from MustReadBlock; want false", ok)
-		} else if len(data) > 0 {
-			err = fmt.Errorf("unexpected non-empty data returned from MustReadBlock: %q", data)
-		}
-		resultCh <- err
-	}()
-	if n := q.GetPendingBytes(); n > 0 {
-		t.Fatalf("pending bytes must be 0; got %d", n)
-	}
-	q.MustClose()
-	select {
-	case err := <-resultCh:
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	case <-time.After(time.Second):
-		t.Fatalf("timeout")
-	}
-}
-
-func TestQueueReadWriteConcurrent(t *testing.T) {
-	path := "queue-read-write-concurrent"
-	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", 0)
-	defer mustDeleteDir(path)
-
-	blocksMap := make(map[string]bool, 1000)
-	var blocksMapLock sync.Mutex
-	blocks := make([]string, 1000)
-	for i := 0; i < 1000; i++ {
-		block := fmt.Sprintf("block #%d", i)
-		blocksMap[block] = true
-		blocks[i] = block
-	}
-
-	// Start block readers
-	var readersWG sync.WaitGroup
-	for workerID := 0; workerID < 10; workerID++ {
-		readersWG.Add(1)
-		go func() {
-			defer readersWG.Done()
-			for {
-				block, ok := q.MustReadBlock(nil)
-				if !ok {
-					return
-				}
-				blocksMapLock.Lock()
-				if !blocksMap[string(block)] {
-					panic(fmt.Errorf("unexpected block read: %q", block))
-				}
-				delete(blocksMap, string(block))
-				blocksMapLock.Unlock()
-			}
-		}()
-	}
-
-	// Start block writers
-	blocksCh := make(chan string)
-	var writersWG sync.WaitGroup
-	for workerID := 0; workerID < 10; workerID++ {
-		writersWG.Add(1)
-		go func(workerID int) {
-			defer writersWG.Done()
-			for block := range blocksCh {
-				q.MustWriteBlock([]byte(block))
-			}
-		}(workerID)
-	}
-	for _, block := range blocks {
-		blocksCh <- block
-	}
-	close(blocksCh)
-
-	// Wait for block writers to finish
-	writersWG.Wait()
-
-	// Notify readers that the queue is closed
-	q.MustClose()
-
-	// Wait for block readers to finish
-	readersWG.Wait()
-
-	// Read the remaining blocks in q.
-	q = MustOpen(path, "foobar", 0)
-	defer q.MustClose()
-	resultCh := make(chan error)
-	go func() {
-		for len(blocksMap) > 0 {
-			block, ok := q.MustReadBlock(nil)
-			if !ok {
-				resultCh <- fmt.Errorf("unexpected ok=false returned from MustReadBlock")
-				return
-			}
-			if !blocksMap[string(block)] {
-				resultCh <- fmt.Errorf("unexpected block read from the queue: %q", block)
-				return
-			}
-			delete(blocksMap, string(block))
-		}
-		resultCh <- nil
-	}()
-	select {
-	case err := <-resultCh:
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatalf("timeout")
-	}
-	if n := q.GetPendingBytes(); n > 0 {
-		t.Fatalf("pending bytes must be 0; got %d", n)
-	}
-}
-
 func TestQueueChunkManagementSimple(t *testing.T) {
 	path := "queue-chunk-management-simple"
 	mustDeleteDir(path)
 	const chunkFileSize = 100
 	const maxBlockSize = 20
-	q := mustOpen(path, "foobar", chunkFileSize, maxBlockSize, 0)
+	q := mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
 	defer mustDeleteDir(path)
 	defer q.MustClose()
 	var blocks []string
@@ -386,7 +263,7 @@ func TestQueueChunkManagementSimple(t *testing.T) {
 		t.Fatalf("unexpected zero number of bytes pending")
 	}
 	for _, block := range blocks {
-		data, ok := q.MustReadBlock(nil)
+		data, ok := q.MustReadBlockNonblocking(nil)
 		if !ok {
 			t.Fatalf("unexpected ok=false")
 		}
@@ -404,7 +281,7 @@ func TestQueueChunkManagementPeriodicClose(t *testing.T) {
 	mustDeleteDir(path)
 	const chunkFileSize = 100
 	const maxBlockSize = 20
-	q := mustOpen(path, "foobar", chunkFileSize, maxBlockSize, 0)
+	q := mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
 	defer func() {
 		q.MustClose()
 		mustDeleteDir(path)
@@ -415,13 +292,13 @@ func TestQueueChunkManagementPeriodicClose(t *testing.T) {
 		q.MustWriteBlock([]byte(block))
 		blocks = append(blocks, block)
 		q.MustClose()
-		q = mustOpen(path, "foobar", chunkFileSize, maxBlockSize, 0)
+		q = mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
 	}
 	if n := q.GetPendingBytes(); n == 0 {
 		t.Fatalf("unexpected zero number of bytes pending")
 	}
 	for _, block := range blocks {
-		data, ok := q.MustReadBlock(nil)
+		data, ok := q.MustReadBlockNonblocking(nil)
 		if !ok {
 			t.Fatalf("unexpected ok=false")
 		}
@@ -429,7 +306,7 @@ func TestQueueChunkManagementPeriodicClose(t *testing.T) {
 			t.Fatalf("unexpected block read; got %q; want %q", data, block)
 		}
 		q.MustClose()
-		q = mustOpen(path, "foobar", chunkFileSize, maxBlockSize, 0)
+		q = mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
 	}
 	if n := q.GetPendingBytes(); n != 0 {
 		t.Fatalf("unexpected non-zero number of pending bytes: %d", n)
@@ -440,7 +317,7 @@ func TestQueueLimitedSize(t *testing.T) {
 	const maxPendingBytes = 1000
 	path := "queue-limited-size"
 	mustDeleteDir(path)
-	q := MustOpen(path, "foobar", maxPendingBytes)
+	q := mustOpen(path, "foobar", maxPendingBytes)
 	defer func() {
 		q.MustClose()
 		mustDeleteDir(path)
@@ -456,7 +333,7 @@ func TestQueueLimitedSize(t *testing.T) {
 	var buf []byte
 	var ok bool
 	for _, block := range blocks {
-		buf, ok = q.MustReadBlock(buf[:0])
+		buf, ok = q.MustReadBlockNonblocking(buf[:0])
 		if !ok {
 			t.Fatalf("unexpected ok=false")
 		}
@@ -473,7 +350,7 @@ func TestQueueLimitedSize(t *testing.T) {
 	if n := q.GetPendingBytes(); n > maxPendingBytes {
 		t.Fatalf("too many pending bytes; got %d; mustn't exceed %d", n, maxPendingBytes)
 	}
-	buf, ok = q.MustReadBlock(buf[:0])
+	buf, ok = q.MustReadBlockNonblocking(buf[:0])
 	if !ok {
 		t.Fatalf("unexpected ok=false")
 	}
diff --git a/lib/persistentqueue/persistentqueue_timing_test.go b/lib/persistentqueue/persistentqueue_timing_test.go
index 02e87513f..e7a3b7874 100644
--- a/lib/persistentqueue/persistentqueue_timing_test.go
+++ b/lib/persistentqueue/persistentqueue_timing_test.go
@@ -2,13 +2,14 @@ package persistentqueue
 
 import (
 	"fmt"
+	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkQueueThroughputSerial(b *testing.B) {
-	const iterationsCount = 10
+	const iterationsCount = 100
 	for _, blockSize := range []int{1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6} {
 		block := make([]byte, blockSize)
 		b.Run(fmt.Sprintf("block-size-%d", blockSize), func(b *testing.B) {
@@ -16,7 +17,7 @@ func BenchmarkQueueThroughputSerial(b *testing.B) {
 			b.SetBytes(int64(blockSize) * iterationsCount)
 			path := fmt.Sprintf("bench-queue-throughput-serial-%d", blockSize)
 			mustDeleteDir(path)
-			q := MustOpen(path, "foobar", 0)
+			q := mustOpen(path, "foobar", 0)
 			defer func() {
 				q.MustClose()
 				mustDeleteDir(path)
@@ -29,7 +30,7 @@ func BenchmarkQueueThroughputSerial(b *testing.B) {
 }
 
 func BenchmarkQueueThroughputConcurrent(b *testing.B) {
-	const iterationsCount = 10
+	const iterationsCount = 100
 	for _, blockSize := range []int{1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6} {
 		block := make([]byte, blockSize)
 		b.Run(fmt.Sprintf("block-size-%d", blockSize), func(b *testing.B) {
@@ -37,28 +38,31 @@ func BenchmarkQueueThroughputConcurrent(b *testing.B) {
 			b.SetBytes(int64(blockSize) * iterationsCount)
 			path := fmt.Sprintf("bench-queue-throughput-concurrent-%d", blockSize)
 			mustDeleteDir(path)
-			q := MustOpen(path, "foobar", 0)
+			q := mustOpen(path, "foobar", 0)
+			var qLock sync.Mutex
 			defer func() {
 				q.MustClose()
 				mustDeleteDir(path)
 			}()
 			b.RunParallel(func(pb *testing.PB) {
 				for pb.Next() {
+					qLock.Lock()
 					writeReadIteration(q, block, iterationsCount)
+					qLock.Unlock()
 				}
 			})
 		})
 	}
 }
 
-func writeReadIteration(q *Queue, block []byte, iterationsCount int) {
+func writeReadIteration(q *queue, block []byte, iterationsCount int) {
 	for i := 0; i < iterationsCount; i++ {
 		q.MustWriteBlock(block)
 	}
 	var ok bool
 	bb := bbPool.Get()
 	for i := 0; i < iterationsCount; i++ {
-		bb.B, ok = q.MustReadBlock(bb.B[:0])
+		bb.B, ok = q.MustReadBlockNonblocking(bb.B[:0])
 		if !ok {
 			panic(fmt.Errorf("unexpected ok=false"))
 		}