Repository: VictoriaMetrics/VictoriaMetrics
Pull request #3656 (2023-01-16 12:04:18)

vmui: give more visually different colors to graph lines
1. Use a pre-defined set of visually different colors for the first 10 lines. 
2. Change the color generation from a string. Makes a more distinguishable color for similar strings.
<details>
  <summary>Example</summary>
  <img width=600 src=https://user-images.githubusercontent.com/29711459/212671115-453eba26-1c3f-4ce6-8a12-0ae828a7a8de.jpg />
  <img src=https://user-images.githubusercontent.com/29711459/212671107-e18e5315-e264-41d5-9d47-91e6355722a9.jpg />
</details>

#3571

codecov[bot]: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Base: **57.58**% // Head: **57.58**% // Decreases project coverage by **`-0.00%`** :warning:
> Coverage data is based on head [(`d1bb162`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) compared to base [(`094ae82`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/094ae82df55a7db17b61a2c1e4c88a0fe685f58f?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).
> Patch has no changes to coverable lines.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #3656      +/-   ##
==========================================
- Coverage   57.58%   57.58%   -0.01%     
==========================================
  Files         309      309              
  Lines       52020    52020              
==========================================
- Hits        29955    29954       -1     
  Misses      20173    20173              
- Partials     1892     1893       +1     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Coverage Δ | |
|---|---|---|
| [app/vmalert/remotewrite/remotewrite.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvcmVtb3Rld3JpdGUvcmVtb3Rld3JpdGUuZ28=) | `62.32% <0.00%> (-2.06%)` | :arrow_down: |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `64.20% <0.00%> (-0.33%)` | :arrow_down: |
| [lib/storage/storage.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-bGliL3N0b3JhZ2Uvc3RvcmFnZS5nbw==) | `70.48% <0.00%> (-0.12%)` | :arrow_down: |
| [lib/storage/partition.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-bGliL3N0b3JhZ2UvcGFydGl0aW9uLmdv) | `63.55% <0.00%> (+0.26%)` | :arrow_up: |
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `86.52% <0.00%> (+1.19%)` | :arrow_up: |

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)

</details>

[:umbrella: View full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/3656?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Do you have feedback about the report comment? [Let us know in this issue](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index d4de82c8c..5c0925f9d 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -646,10 +646,6 @@ func registerStorageMetrics(strg *storage.Storage) {
 		return float64(m().TooSmallTimestampRows)
 	})
 
-	metrics.NewGauge(`vm_search_delays_total`, func() float64 {
-		return float64(m().SearchDelays)
-	})
-
 	metrics.NewGauge(`vm_slow_row_inserts_total`, func() float64 {
 		return float64(m().SlowRowInserts)
 	})
diff --git a/app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx b/app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx
index d9028cf7c..905adcd74 100644
--- a/app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx
+++ b/app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx
@@ -1,9 +1,9 @@
-import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
+import React, { FC, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
 import { MetricResult } from "../../../api/types";
 import LineChart from "../../Chart/LineChart/LineChart";
 import { AlignedData as uPlotData, Series as uPlotSeries } from "uplot";
 import Legend from "../../Chart/Legend/Legend";
-import { getHideSeries, getLegendItem, getSeriesItem } from "../../../utils/uplot/series";
+import { getHideSeries, getLegendItem, getSeriesItemContext } from "../../../utils/uplot/series";
 import { getLimitsYAxis, getMinMaxBuffer, getTimeSeries } from "../../../utils/uplot/axes";
 import { LegendItemType } from "../../../utils/uplot/types";
 import { TimeParams } from "../../../types";
@@ -59,6 +59,7 @@ const GraphView: FC<GraphViewProps> = ({
 }) => {
   const { timezone } = useTimeState();
   const currentStep = useMemo(() => customStep || period.step || "1s", [period.step, customStep]);
+  const getSeriesItem = useCallback(getSeriesItemContext(), [data]);
 
   const [dataChart, setDataChart] = useState<uPlotData>([[]]);
   const [series, setSeries] = useState<uPlotSeries[]>([]);
diff --git a/app/vmui/packages/vmui/src/pages/TopQueries/index.tsx b/app/vmui/packages/vmui/src/pages/TopQueries/index.tsx
index 752f8c1ee..ef269d830 100644
--- a/app/vmui/packages/vmui/src/pages/TopQueries/index.tsx
+++ b/app/vmui/packages/vmui/src/pages/TopQueries/index.tsx
@@ -40,7 +40,7 @@ const Index: FC = () => {
   const getQueryStatsTitle = (key: keyof TopQueryStats) => {
     if (!data) return key;
     const value = data[key];
-    if (typeof value === "number") return formatPrettyNumber(value, 0, value);
+    if (typeof value === "number") return formatPrettyNumber(value);
     return value || key;
   };
 
diff --git a/app/vmui/packages/vmui/src/utils/color.ts b/app/vmui/packages/vmui/src/utils/color.ts
index fc68d7deb..e1e8b0411 100644
--- a/app/vmui/packages/vmui/src/utils/color.ts
+++ b/app/vmui/packages/vmui/src/utils/color.ts
@@ -1,14 +1,35 @@
-export const getColorFromString = (str: string): string => {
-  let hash = 0;
-  for (let i = 0; i < str.length; i++) {
-    hash = str.charCodeAt(i) + ((hash << 5) - hash);
-  }
-  let colour = "#";
-  for (let i = 0; i < 3; i++) {
-    const value = (hash >> (i * 8)) & 0xFF;
-    colour += ("00" + value.toString(16)).substr(-2);
+export const baseContrastColors = [
+  "#e54040",
+  "#32a9dc",
+  "#2ee329",
+  "#7126a1",
+  "#e38f0f",
+  "#3d811a",
+  "#ffea00",
+  "#2d2d2d",
+  "#da42a6",
+  "#a44e0c",
+];
+
+export const getColorFromString = (text: string): string => {
+  const SEED = 16777215;
+  const FACTOR = 49979693;
+
+  let b = 1;
+  let d = 0;
+  let f = 1;
+
+  if (text.length > 0) {
+    for (let i = 0; i < text.length; i++) {
+      text[i].charCodeAt(0) > d && (d = text[i].charCodeAt(0));
+      f = parseInt(String(SEED / d));
+      b = (b + text[i].charCodeAt(0) * f * FACTOR) % SEED;
+    }
   }
-  return colour;
+
+  let hex = ((b * text.length) % SEED).toString(16);
+  hex = hex.padEnd(6, hex);
+  return `#${hex}`;
 };
 
 export const hexToRGB = (hex: string): string => {
diff --git a/app/vmui/packages/vmui/src/utils/uplot/helpers.ts b/app/vmui/packages/vmui/src/utils/uplot/helpers.ts
index d55fa54cb..b095c86f7 100644
--- a/app/vmui/packages/vmui/src/utils/uplot/helpers.ts
+++ b/app/vmui/packages/vmui/src/utils/uplot/helpers.ts
@@ -1,5 +1,4 @@
 import uPlot, { Axis } from "uplot";
-import { getColorFromString } from "../color";
 
 export const defaultOptions = {
   legend: {
@@ -37,7 +36,18 @@ export const formatPrettyNumber = (n: number | null | undefined, min = 0, max =
   if (n === undefined || n === null) {
     return "";
   }
-  let digits = 3 + Math.floor(1 + Math.log10(Math.max(Math.abs(min), Math.abs(max))) - Math.log10(Math.abs(min - max)));
+  const range = Math.abs(max - min);
+  if (isNaN(range) || range == 0) {
+    // Return the constant number as is if the range isn't set of it is too small.
+    if (Math.abs(n) >= 1000) {
+      return n.toLocaleString("en-US");
+    }
+    return n.toString();
+  }
+  // Make sure n has 3 significant digits on the given range.
+  // This precision should be enough for most UX cases,
+  // since the remaining digits are usually a white noise.
+  let digits = 3 + Math.floor(1 + Math.log10(Math.max(Math.abs(min), Math.abs(max))) - Math.log10(range));
   if (isNaN(digits) || digits > 20) {
     digits = 20;
   }
@@ -74,6 +84,4 @@ export const sizeAxis = (u: uPlot, values: string[], axisIdx: number, cycleNum:
   return Math.ceil(axisSize);
 };
 
-export const getColorLine = (label: string): string => getColorFromString(label);
-
 export const getDashLine = (group: number): number[] => group <= 1 ? [] : [group*4, group*1.2];
diff --git a/app/vmui/packages/vmui/src/utils/uplot/series.ts b/app/vmui/packages/vmui/src/utils/uplot/series.ts
index 31e8c149f..2314a4f1c 100644
--- a/app/vmui/packages/vmui/src/utils/uplot/series.ts
+++ b/app/vmui/packages/vmui/src/utils/uplot/series.ts
@@ -2,26 +2,34 @@ import { MetricResult } from "../../api/types";
 import { Series } from "uplot";
 import { getNameForMetric } from "../metric";
 import { BarSeriesItem, Disp, Fill, LegendItemType, Stroke } from "./types";
-import { getColorLine } from "./helpers";
 import { HideSeriesArgs } from "./types";
+import { baseContrastColors, getColorFromString } from "../color";
 
 interface SeriesItem extends Series {
   freeFormFields: {[key: string]: string};
 }
 
-export const getSeriesItem = (d: MetricResult, hideSeries: string[], alias: string[]): SeriesItem => {
-  const label = getNameForMetric(d, alias[d.group - 1]);
-  return {
-    label,
-    freeFormFields: d.metric,
-    width: 1.4,
-    stroke: getColorLine(label),
-    show: !includesHideSeries(label, hideSeries),
-    scale: "1",
-    points: {
-      size: 4.2,
-      width: 1.4
-    }
+export const getSeriesItemContext = () => {
+  const colorState: {[key: string]: string} = {};
+
+  return (d: MetricResult, hideSeries: string[], alias: string[]): SeriesItem => {
+    const label = getNameForMetric(d, alias[d.group - 1]);
+    const countSavedColors = Object.keys(colorState).length;
+    const hasBasicColors = countSavedColors < baseContrastColors.length;
+    if (hasBasicColors) colorState[label] = colorState[label] || baseContrastColors[countSavedColors];
+
+    return {
+      label,
+      freeFormFields: d.metric,
+      width: 1.4,
+      stroke: colorState[label] || getColorFromString(label),
+      show: !includesHideSeries(label, hideSeries),
+      scale: "1",
+      points: {
+        size: 4.2,
+        width: 1.4
+      }
+    };
   };
 };
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8fadc6930..76fb75647 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -17,12 +17,18 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to show custom dashboards at vmui by specifying a path to a directory with dashboard config files via `-vmui.customDashboardsPath` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3322) and [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): apply the `step` globally to all the displayed graphs. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3574).
+* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve the appearance of graph lines by using more visually distinct colors. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3656).
 
+* BUGFIX: do not slow down concurrently executed queries during assisted merges, since assisted merges already prioritize data ingestion over queries. The probability of assisted merges has been increased starting from [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850) because of internal refactoring. This could result in slowed down queries when there is a plenty of free CPU resources. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3647) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3641) issues.
 * BUGFIX: reduce the increased CPU usage at `vmselect` to v1.85.3 level when processing heavy queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3641).
+* BUGFIX: [retention filters](https://docs.victoriametrics.com/#retention-filters): fix `FATAL: cannot locate metric name for metricID=...: EOF` panic, which could occur when retention filters are enabled.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): [dockerswarm_sd_configs](https://docs.victoriametrics.com/sd_configs.html#dockerswarm_sd_configs): apply `filters` only to objects of the specified `role`. Previously filters were applied to all the objects, which could cause errors when different types of objects were used with filters that were not compatible with them. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3579).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): suppress all the scrape errors when `-promscrape.suppressScrapeErrors` is enabled. Previously some scrape errors were logged even if `-promscrape.suppressScrapeErrors` flag was set.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently put the scrape url with scrape target labels to all error logs for failed scrapes. Previously some failed scrapes were logged without this information.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not send stale markers to remote storage for series exceeding the configured [series limit](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply [series limit](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter) when [staleness tracking](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) is disabled.
 * BUGFIX: [Pushgateway import](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format): properly return `200 OK` HTTP response code. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly parse `M` and `Mi` suffixes as `1e6` multipliers in `1M` and `1Mi` numeric constants. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664). The issue has been introduced in [v1.86.0](https://docs.victoriametrics.com/CHANGELOG.html#v1860).
 
 ## [v1.86.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.86.1)
 
diff --git a/go.mod b/go.mod
index 92937c54e..04b44b322 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.1.0
 	github.com/VictoriaMetrics/metrics v1.23.1
-	github.com/VictoriaMetrics/metricsql v0.51.1
+	github.com/VictoriaMetrics/metricsql v0.51.2
 	github.com/aws/aws-sdk-go-v2 v1.17.3
 	github.com/aws/aws-sdk-go-v2/config v1.18.8
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47
diff --git a/go.sum b/go.sum
index b92d01b20..eebce8771 100644
--- a/go.sum
+++ b/go.sum
@@ -69,8 +69,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
 github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
 github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0=
 github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
-github.com/VictoriaMetrics/metricsql v0.51.1 h1:gmh3ZGCDrqUTdhUrr87eJOXMOputDYs1PtLwTfySTsI=
-github.com/VictoriaMetrics/metricsql v0.51.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
+github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI=
+github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go
index cb875c026..158b96cd4 100644
--- a/lib/mergeset/table.go
+++ b/lib/mergeset/table.go
@@ -18,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 )
 
@@ -782,11 +781,8 @@ func (tb *Table) assistedMergeForInmemoryParts() {
 			return
 		}
 
-		// Prioritize assisted merges over searches.
-		storagepacelimiter.Search.Inc()
 		atomic.AddUint64(&tb.inmemoryAssistedMerges, 1)
 		err := tb.mergeInmemoryParts()
-		storagepacelimiter.Search.Dec()
 		if err == nil {
 			continue
 		}
@@ -806,11 +802,8 @@ func (tb *Table) assistedMergeForFileParts() {
 			return
 		}
 
-		// Prioritize assisted merges over searches.
-		storagepacelimiter.Search.Inc()
 		atomic.AddUint64(&tb.fileAssistedMerges, 1)
 		err := tb.mergeExistingParts(false)
-		storagepacelimiter.Search.Dec()
 		if err == nil {
 			continue
 		}
diff --git a/lib/pacelimiter/pacelimiter.go b/lib/pacelimiter/pacelimiter.go
deleted file mode 100644
index 462a0ef6c..000000000
--- a/lib/pacelimiter/pacelimiter.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package pacelimiter
-
-import (
-	"sync"
-	"sync/atomic"
-)
-
-// PaceLimiter throttles WaitIfNeeded callers while the number of Inc calls is bigger than the number of Dec calls.
-//
-// It is expected that Inc is called before performing high-priority work,
-// while Dec is called when the work is done.
-// WaitIfNeeded must be called inside the work which must be throttled (i.e. lower-priority work).
-// It may be called in the loop before performing a part of low-priority work.
-type PaceLimiter struct {
-	mu          sync.Mutex
-	cond        *sync.Cond
-	delaysTotal uint64
-	n           int32
-}
-
-// New returns pace limiter that throttles WaitIfNeeded callers while the number of Inc calls is bigger than the number of Dec calls.
-func New() *PaceLimiter {
-	var pl PaceLimiter
-	pl.cond = sync.NewCond(&pl.mu)
-	return &pl
-}
-
-// Inc increments pl.
-func (pl *PaceLimiter) Inc() {
-	atomic.AddInt32(&pl.n, 1)
-}
-
-// Dec decrements pl.
-func (pl *PaceLimiter) Dec() {
-	if atomic.AddInt32(&pl.n, -1) == 0 {
-		// Wake up all the goroutines blocked in WaitIfNeeded,
-		// since the number of Dec calls equals the number of Inc calls.
-		pl.cond.Broadcast()
-	}
-}
-
-// WaitIfNeeded blocks while the number of Inc calls is bigger than the number of Dec calls.
-func (pl *PaceLimiter) WaitIfNeeded() {
-	if atomic.LoadInt32(&pl.n) <= 0 {
-		// Fast path - there is no need in lock.
-		return
-	}
-	// Slow path - wait until Dec is called.
-	pl.mu.Lock()
-	for atomic.LoadInt32(&pl.n) > 0 {
-		pl.delaysTotal++
-		pl.cond.Wait()
-	}
-	pl.mu.Unlock()
-}
-
-// DelaysTotal returns the number of delays inside WaitIfNeeded.
-func (pl *PaceLimiter) DelaysTotal() uint64 {
-	pl.mu.Lock()
-	n := pl.delaysTotal
-	pl.mu.Unlock()
-	return n
-}
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index b4d5317f7..dd64733cc 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -41,9 +41,10 @@ import (
 )
 
 var (
-	noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
-	strictParse    = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
-	dryRun         = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
+	noStaleMarkers       = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
+	seriesLimitPerTarget = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
+	strictParse          = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
+	dryRun               = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
 		"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
 		"See also -promscrape.config.strictParse command-line flag. "+
 		"Pass -loggerLevel=ERROR if you don't need to see info messages in the output.")
@@ -971,6 +972,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if sc.NoStaleMarkers != nil {
 		noStaleTracking = *sc.NoStaleMarkers
 	}
+	seriesLimit := *seriesLimitPerTarget
+	if sc.SeriesLimit > 0 {
+		seriesLimit = sc.SeriesLimit
+	}
 	swc := &scrapeWorkConfig{
 		scrapeInterval:       scrapeInterval,
 		scrapeIntervalString: scrapeInterval.String(),
@@ -995,7 +1000,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		streamParse:          sc.StreamParse,
 		scrapeAlignInterval:  sc.ScrapeAlignInterval.Duration(),
 		scrapeOffset:         sc.ScrapeOffset.Duration(),
-		seriesLimit:          sc.SeriesLimit,
+		seriesLimit:          seriesLimit,
 		noStaleMarkers:       noStaleTracking,
 	}
 	return swc, nil
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index ed4c77018..2370869db 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -37,7 +37,6 @@ var (
 		"See also -promscrape.suppressScrapeErrorsDelay")
 	suppressScrapeErrorsDelay = flag.Duration("promscrape.suppressScrapeErrorsDelay", 0, "The delay for suppressing repeated scrape errors logging per each scrape targets. "+
 		"This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors")
-	seriesLimitPerTarget          = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
 	minResponseSizeForStreamParse = flagutil.NewBytes("promscrape.minResponseSizeForStreamParse", 1e6, "The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode")
 )
 
@@ -451,7 +450,7 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
 	wc := writeRequestCtxPool.Get(sw.prevLabelsLen)
 	lastScrape := sw.loadLastScrape()
 	bodyString := bytesutil.ToUnsafeString(body.B)
-	areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
+	areIdenticalSeries := sw.areIdenticalSeries(lastScrape, bodyString)
 	if err != nil {
 		up = 0
 		scrapesFailed.Inc()
@@ -485,9 +484,6 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
 	samplesDropped := 0
 	if sw.seriesLimitExceeded || !areIdenticalSeries {
 		samplesDropped = sw.applySeriesLimit(wc)
-		if samplesDropped > 0 {
-			sw.seriesLimitExceeded = true
-		}
 	}
 	am := &autoMetrics{
 		up:                        up,
@@ -577,7 +573,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 		err = sbr.Init(sr)
 		if err == nil {
 			bodyString = bytesutil.ToUnsafeString(sbr.body)
-			areIdenticalSeries = sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
+			areIdenticalSeries = sw.areIdenticalSeries(lastScrape, bodyString)
 			err = parser.ParseStream(&sbr, scrapeTimestamp, false, func(rows []parser.Row) error {
 				mu.Lock()
 				defer mu.Unlock()
@@ -594,9 +590,6 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 				}
 				if sw.seriesLimitExceeded || !areIdenticalSeries {
 					samplesDropped += sw.applySeriesLimit(wc)
-					if samplesDropped > 0 && !sw.seriesLimitExceeded {
-						sw.seriesLimitExceeded = true
-					}
 				}
 				// Push the collected rows to sw before returning from the callback, since they cannot be held
 				// after returning from the callback - this will result in data race.
@@ -655,6 +648,15 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	return err
 }
 
+func (sw *scrapeWork) areIdenticalSeries(prevData, currData string) bool {
+	if sw.Config.NoStaleMarkers && sw.Config.SeriesLimit <= 0 {
+		// Do not spend CPU time on tracking the changes in series if stale markers are disabled.
+		// The check for series_limit is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660
+		return true
+	}
+	return parser.AreIdenticalSeriesFast(prevData, currData)
+}
+
 // leveledWriteRequestCtxPool allows reducing memory usage when writeRequesCtx
 // structs contain mixed number of labels.
 //
@@ -738,17 +740,13 @@ func (sw *scrapeWork) getSeriesAdded(lastScrape, currScrape string) int {
 }
 
 func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
-	seriesLimit := *seriesLimitPerTarget
-	if sw.Config.SeriesLimit > 0 {
-		seriesLimit = sw.Config.SeriesLimit
+	if sw.Config.SeriesLimit <= 0 {
+		return 0
 	}
-	if sw.seriesLimiter == nil && seriesLimit > 0 {
-		sw.seriesLimiter = bloomfilter.NewLimiter(seriesLimit, 24*time.Hour)
+	if sw.seriesLimiter == nil {
+		sw.seriesLimiter = bloomfilter.NewLimiter(sw.Config.SeriesLimit, 24*time.Hour)
 	}
 	sl := sw.seriesLimiter
-	if sl == nil {
-		return 0
-	}
 	dstSeries := wc.writeRequest.Timeseries[:0]
 	samplesDropped := 0
 	for _, ts := range wc.writeRequest.Timeseries {
@@ -761,6 +759,9 @@ func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
 	}
 	prompbmarshal.ResetTimeSeries(wc.writeRequest.Timeseries[len(dstSeries):])
 	wc.writeRequest.Timeseries = dstSeries
+	if samplesDropped > 0 && !sw.seriesLimitExceeded {
+		sw.seriesLimitExceeded = true
+	}
 	return samplesDropped
 }
 
@@ -784,6 +785,13 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
 		am := &autoMetrics{}
 		sw.addAutoMetrics(am, wc, timestamp)
 	}
+
+	// Apply series limit to stale markers in order to prevent sending stale markers for newly created series.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660
+	if sw.seriesLimitExceeded {
+		sw.applySeriesLimit(wc)
+	}
+
 	series := wc.writeRequest.Timeseries
 	if len(series) == 0 {
 		return
diff --git a/lib/storage/partition.go b/lib/storage/partition.go
index cd0358337..8fcc8bc13 100644
--- a/lib/storage/partition.go
+++ b/lib/storage/partition.go
@@ -20,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 )
 
@@ -610,14 +609,8 @@ func (pt *partition) assistedMergeForInmemoryParts() {
 			return
 		}
 
-		// There are too many unmerged inmemory parts.
-		// This usually means that the app cannot keep up with the data ingestion rate.
-		// Assist with mering inmemory parts.
-		// Prioritize assisted merges over searches.
-		storagepacelimiter.Search.Inc()
 		atomic.AddUint64(&pt.inmemoryAssistedMerges, 1)
 		err := pt.mergeInmemoryParts()
-		storagepacelimiter.Search.Dec()
 		if err == nil {
 			continue
 		}
@@ -637,14 +630,8 @@ func (pt *partition) assistedMergeForSmallParts() {
 			return
 		}
 
-		// There are too many unmerged small parts.
-		// This usually means that the app cannot keep up with the data ingestion rate.
-		// Assist with mering small parts.
-		// Prioritize assisted merges over searches.
-		storagepacelimiter.Search.Inc()
 		atomic.AddUint64(&pt.smallAssistedMerges, 1)
 		err := pt.mergeExistingParts(false)
-		storagepacelimiter.Search.Dec()
 		if err == nil {
 			continue
 		}
diff --git a/lib/storage/search.go b/lib/storage/search.go
index 06252596d..29a4b5b90 100644
--- a/lib/storage/search.go
+++ b/lib/storage/search.go
@@ -10,7 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
 )
 
 // BlockRef references a Block.
@@ -448,7 +447,6 @@ func checkSearchDeadlineAndPace(deadline uint64) error {
 	if fasttime.UnixTimestamp() > deadline {
 		return ErrDeadlineExceeded
 	}
-	storagepacelimiter.Search.WaitIfNeeded()
 	return nil
 }
 
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index f3514080d..1a14f470d 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -26,7 +26,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 	"github.com/VictoriaMetrics/fastcache"
@@ -446,8 +445,6 @@ type Metrics struct {
 	TooSmallTimestampRows uint64
 	TooBigTimestampRows   uint64
 
-	SearchDelays uint64
-
 	SlowRowInserts         uint64
 	SlowPerDayIndexInserts uint64
 	SlowMetricNameLoads    uint64
@@ -517,8 +514,6 @@ func (s *Storage) UpdateMetrics(m *Metrics) {
 	m.TooSmallTimestampRows += atomic.LoadUint64(&s.tooSmallTimestampRows)
 	m.TooBigTimestampRows += atomic.LoadUint64(&s.tooBigTimestampRows)
 
-	m.SearchDelays = storagepacelimiter.Search.DelaysTotal()
-
 	m.SlowRowInserts += atomic.LoadUint64(&s.slowRowInserts)
 	m.SlowPerDayIndexInserts += atomic.LoadUint64(&s.slowPerDayIndexInserts)
 	m.SlowMetricNameLoads += atomic.LoadUint64(&s.slowMetricNameLoads)
diff --git a/lib/storagepacelimiter/storagepacelimiter.go b/lib/storagepacelimiter/storagepacelimiter.go
deleted file mode 100644
index e309e6c52..000000000
--- a/lib/storagepacelimiter/storagepacelimiter.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package storagepacelimiter
-
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pacelimiter"
-)
-
-// Search limits the pace of search calls when there is at least a single in-flight assisted merge.
-//
-// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/291
-var Search = pacelimiter.New()
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
index c3aebd80d..f0dec0a82 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go
@@ -289,6 +289,9 @@ func scanPositiveNumber(s string) (string, error) {
 }
 
 func scanNumMultiplier(s string) int {
+	if len(s) > 3 {
+		s = s[:3]
+	}
 	s = strings.ToLower(s)
 	switch true {
 	case strings.HasPrefix(s, "kib"):
@@ -616,7 +619,6 @@ func scanSingleDuration(s string, canBeNegative bool) int {
 	if len(s) == 0 {
 		return -1
 	}
-	s = strings.ToLower(s)
 	i := 0
 	if s[0] == '-' && canBeNegative {
 		i++
@@ -637,14 +639,26 @@ func scanSingleDuration(s string, canBeNegative bool) int {
 			return -1
 		}
 	}
-	switch s[i] {
+	switch unicode.ToLower(rune(s[i])) {
 	case 'm':
-		if i+1 < len(s) && s[i+1] == 's' {
-			// duration in ms
-			return i + 2
+		if i+1 < len(s) {
+			switch unicode.ToLower(rune(s[i+1])) {
+			case 's':
+				// duration in ms
+				return i + 2
+			case 'i', 'b':
+				// This is not a duration, but Mi or MB suffix.
+				// See parsePositiveNumber() and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664
+				return -1
+			}
 		}
-		// duration in minutes
-		return i + 1
+		// Allow small m for durtion in minutes.
+		// Big M means 1e6.
+		// See parsePositiveNumber() and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664
+		if s[i] == 'm' {
+			return i + 1
+		}
+		return -1
 	case 's', 'h', 'd', 'w', 'y', 'i':
 		return i + 1
 	default:
diff --git a/vendor/modules.txt b/vendor/modules.txt
index bf77ddb3d..e32545b3c 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -70,7 +70,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.23.1
 ## explicit; go 1.15
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.51.1
+# github.com/VictoriaMetrics/metricsql v0.51.2
 ## explicit; go 1.13
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop


Test patch:
diff --git a/lib/pacelimiter/pacelimiter_test.go b/lib/pacelimiter/pacelimiter_test.go
deleted file mode 100644
index 4c5bbf252..000000000
--- a/lib/pacelimiter/pacelimiter_test.go
+++ /dev/null
@@ -1,144 +0,0 @@
-package pacelimiter
-
-import (
-	"fmt"
-	"runtime"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestPacelimiter(t *testing.T) {
-	t.Run("nonblocking", func(t *testing.T) {
-		pl := New()
-		ch := make(chan struct{}, 10)
-		for i := 0; i < cap(ch); i++ {
-			go func() {
-				for j := 0; j < 10; j++ {
-					pl.WaitIfNeeded()
-					runtime.Gosched()
-				}
-				ch <- struct{}{}
-			}()
-		}
-
-		// Check that all the goroutines are finished.
-		timeoutCh := time.After(5 * time.Second)
-		for i := 0; i < cap(ch); i++ {
-			select {
-			case <-ch:
-			case <-timeoutCh:
-				t.Fatalf("timeout")
-			}
-		}
-		if n := pl.DelaysTotal(); n > 0 {
-			t.Fatalf("unexpected non-zero number of delays: %d", n)
-		}
-	})
-	t.Run("blocking", func(t *testing.T) {
-		pl := New()
-		pl.Inc()
-		ch := make(chan struct{}, 10)
-		var wg sync.WaitGroup
-		for i := 0; i < cap(ch); i++ {
-			wg.Add(1)
-			go func() {
-				wg.Done()
-				for j := 0; j < 10; j++ {
-					pl.WaitIfNeeded()
-				}
-				ch <- struct{}{}
-			}()
-		}
-
-		// Check that all the goroutines created above are started and blocked in WaitIfNeeded
-		wg.Wait()
-		select {
-		case <-ch:
-			t.Fatalf("the pl must be blocked")
-		default:
-		}
-
-		// Unblock goroutines and check that they are unblocked.
-		pl.Dec()
-		timeoutCh := time.After(5 * time.Second)
-		for i := 0; i < cap(ch); i++ {
-			select {
-			case <-ch:
-			case <-timeoutCh:
-				t.Fatalf("timeout")
-			}
-		}
-		if n := pl.DelaysTotal(); n == 0 {
-			t.Fatalf("expecting non-zero number of delays")
-		}
-		// Verify that the pl is unblocked now.
-		pl.WaitIfNeeded()
-
-		// Verify that negative count doesn't block pl.
-		pl.Dec()
-		pl.WaitIfNeeded()
-		if n := pl.DelaysTotal(); n == 0 {
-			t.Fatalf("expecting non-zero number of delays after subsequent pl.Dec()")
-		}
-	})
-	t.Run("negative_count", func(t *testing.T) {
-		n := 10
-		pl := New()
-		for i := 0; i < n; i++ {
-			pl.Dec()
-		}
-
-		doneCh := make(chan error)
-		go func() {
-			defer close(doneCh)
-			for i := 0; i < n; i++ {
-				pl.Inc()
-				pl.WaitIfNeeded()
-				if n := pl.DelaysTotal(); n != 0 {
-					doneCh <- fmt.Errorf("expecting zero number of delays")
-					return
-				}
-			}
-			doneCh <- nil
-		}()
-
-		select {
-		case err := <-doneCh:
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-		case <-time.After(5 * time.Second):
-			t.Fatalf("timeout")
-		}
-	})
-	t.Run("concurrent_inc_dec", func(t *testing.T) {
-		pl := New()
-		ch := make(chan struct{}, 10)
-		for i := 0; i < cap(ch); i++ {
-			go func() {
-				for j := 0; j < 10; j++ {
-					pl.Inc()
-					runtime.Gosched()
-					pl.Dec()
-				}
-				ch <- struct{}{}
-			}()
-		}
-
-		// Verify that all the goroutines are finished
-		timeoutCh := time.After(5 * time.Second)
-		for i := 0; i < cap(ch); i++ {
-			select {
-			case <-ch:
-			case <-timeoutCh:
-				t.Fatalf("timeout")
-			}
-		}
-		// Verify that the pl is unblocked.
-		pl.WaitIfNeeded()
-		if n := pl.DelaysTotal(); n > 0 {
-			t.Fatalf("expecting zer number of delays; got %d", n)
-		}
-	})
-}