Repository: VictoriaMetrics/VictoriaMetrics
Pull request #3857 (2023-02-22 17:02:42)

fix interpolate function for filling only intermediate gaps (#3816)
From now on, `interpolate` function fills only intermediate gaps and doesn't extrapolate the trailing empty values at the beginning and at the end of the time series, what corresponds to its description in the [documentation](https://docs.victoriametrics.com/MetricsQL.html#interpolate).
The previous behavior can be achieved by using the functions `keep_next_value` and `keep_last_value`.

[issue #3816](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3816).


Patch:
diff --git a/README.md b/README.md
index 827583273..919ca1c31 100644
--- a/README.md
+++ b/README.md
@@ -2339,9 +2339,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
-     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 1m0s)
-  -promscrape.kuma.waitTime duration
-     Wait time used by Kuma service discovery. Default value is used if not set
+     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 486055ecf..ac6abf1be 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -1372,9 +1372,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
-     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 1m0s)
-  -promscrape.kuma.waitTime duration
-     Wait time used by Kuma service discovery. Default value is used if not set
+     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index 2bc506d5f..1197d59aa 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -1169,7 +1169,8 @@ func transformInterpolate(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	rvs := args[0]
 	for _, ts := range rvs {
-		values := ts.Values
+		values := skipLeadingNaNs(ts.Values)
+		values = skipTrailingNaNs(values)
 		if len(values) == 0 {
 			continue
 		}
diff --git a/app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx b/app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx
index 623eba300..24e6a72bf 100644
--- a/app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx
+++ b/app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx
@@ -1,6 +1,5 @@
 import React, { FC, useMemo } from "preact/compat";
 import { ExecutionControls } from "../../Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
 import { TimeSelector } from "../../Configurators/TimeRangeSettings/TimeSelector/TimeSelector";
 import GlobalSettings from "../../Configurators/GlobalSettings/GlobalSettings";
 import { useLocation, useNavigate } from "react-router-dom";
@@ -43,15 +42,14 @@ const Header: FC = () => {
   }, [primaryColor]);
 
   const navigate = useNavigate();
-  const { search, pathname } = useLocation();
+  const { pathname } = useLocation();
 
   const headerSetup = useMemo(() => {
     return ((routerOptions[pathname] || {}) as RouterOptions).header || {};
   }, [pathname]);
 
   const onClickLogo = () => {
-    navigate({ pathname: router.home, search: search });
-    setQueryStringWithoutPageReload({});
+    navigate({ pathname: router.home });
     window.location.reload();
   };
 
diff --git a/app/vmui/packages/vmui/src/pages/CardinalityPanel/hooks/useSetQueryParams.ts b/app/vmui/packages/vmui/src/pages/CardinalityPanel/hooks/useSetQueryParams.ts
index 1d69d56f0..abcdf5354 100644
--- a/app/vmui/packages/vmui/src/pages/CardinalityPanel/hooks/useSetQueryParams.ts
+++ b/app/vmui/packages/vmui/src/pages/CardinalityPanel/hooks/useSetQueryParams.ts
@@ -1,10 +1,11 @@
 import { useEffect } from "react";
 import { useCardinalityState } from "../../../state/cardinality/CardinalityStateContext";
 import { compactObject } from "../../../utils/object";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
+import { useSearchParams } from "react-router-dom";
 
 export const useSetQueryParams = () => {
   const { topN, match, date, focusLabel, extraLabel } = useCardinalityState();
+  const [, setSearchParams] = useSearchParams();
 
   const setSearchParamsFromState = () => {
     const params = compactObject({
@@ -15,7 +16,7 @@ export const useSetQueryParams = () => {
       focusLabel,
     });
 
-    setQueryStringWithoutPageReload(params);
+    setSearchParams(params as Record<string, string>);
   };
 
   useEffect(setSearchParamsFromState, [topN, match, date, focusLabel, extraLabel]);
diff --git a/app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts b/app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts
index 8c51aad28..0b3f22feb 100644
--- a/app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts
+++ b/app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts
@@ -4,9 +4,9 @@ import { useCustomPanelState } from "../../../state/customPanel/CustomPanelState
 import { useAppState } from "../../../state/common/StateContext";
 import { useQueryState } from "../../../state/query/QueryStateContext";
 import { displayTypeTabs } from "../DisplayTypeSwitch";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
 import { compactObject } from "../../../utils/object";
 import { useGraphState } from "../../../state/graph/GraphStateContext";
+import { useSearchParams } from "react-router-dom";
 
 export const useSetQueryParams = () => {
   const { tenantId } = useAppState();
@@ -14,6 +14,7 @@ export const useSetQueryParams = () => {
   const { query } = useQueryState();
   const { duration, relativeTime, period: { date, step } } = useTimeState();
   const { customStep } = useGraphState();
+  const [, setSearchParams] = useSearchParams();
 
   const setSearchParamsFromState = () => {
     const params: Record<string, unknown> = {};
@@ -29,7 +30,7 @@ export const useSetQueryParams = () => {
       if ((step !== customStep) && customStep) params[`${group}.step_input`] = customStep;
     });
 
-    setQueryStringWithoutPageReload(compactObject(params));
+    setSearchParams(compactObject(params) as Record<string, string>);
   };
 
   useEffect(setSearchParamsFromState, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);
diff --git a/app/vmui/packages/vmui/src/pages/ExploreMetrics/hooks/useSetQueryParams.ts b/app/vmui/packages/vmui/src/pages/ExploreMetrics/hooks/useSetQueryParams.ts
index c0fe6d70b..cea591cc5 100644
--- a/app/vmui/packages/vmui/src/pages/ExploreMetrics/hooks/useSetQueryParams.ts
+++ b/app/vmui/packages/vmui/src/pages/ExploreMetrics/hooks/useSetQueryParams.ts
@@ -1,8 +1,8 @@
 import { useEffect } from "react";
 import { compactObject } from "../../../utils/object";
 import { useTimeState } from "../../../state/time/TimeStateContext";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
 import { useGraphState } from "../../../state/graph/GraphStateContext";
+import { useSearchParams } from "react-router-dom";
 
 interface queryProps {
   job: string
@@ -14,6 +14,7 @@ interface queryProps {
 export const useSetQueryParams = ({ job, instance, metrics, size }: queryProps) => {
   const { duration, relativeTime, period: { date } } = useTimeState();
   const { customStep } = useGraphState();
+  const [, setSearchParams] = useSearchParams();
 
   const setSearchParamsFromState = () => {
     const params = compactObject({
@@ -27,7 +28,7 @@ export const useSetQueryParams = ({ job, instance, metrics, size }: queryProps)
       metrics
     });
 
-    setQueryStringWithoutPageReload(params);
+    setSearchParams(params);
   };
 
   useEffect(setSearchParamsFromState, [duration, relativeTime, date, customStep, job, instance, metrics, size]);
diff --git a/app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useSetQueryParams.ts b/app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useSetQueryParams.ts
index da901be97..4ee814cc4 100644
--- a/app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useSetQueryParams.ts
+++ b/app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useSetQueryParams.ts
@@ -1,12 +1,13 @@
 import { useEffect } from "react";
 import { compactObject } from "../../../utils/object";
 import { useTimeState } from "../../../state/time/TimeStateContext";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
 import { useGraphState } from "../../../state/graph/GraphStateContext";
+import { useSearchParams } from "react-router-dom";
 
 export const useSetQueryParams = () => {
   const { duration, relativeTime, period: { date } } = useTimeState();
   const { customStep } = useGraphState();
+  const [, setSearchParams] = useSearchParams();
 
   const setSearchParamsFromState = () => {
     const params = compactObject({
@@ -16,7 +17,7 @@ export const useSetQueryParams = () => {
       ["g0.relative_time"]: relativeTime
     });
 
-    setQueryStringWithoutPageReload(params);
+    setSearchParams(params);
   };
 
   useEffect(setSearchParamsFromState, [duration, relativeTime, date, customStep]);
diff --git a/app/vmui/packages/vmui/src/pages/TopQueries/hooks/useSetQueryParams.ts b/app/vmui/packages/vmui/src/pages/TopQueries/hooks/useSetQueryParams.ts
index 712d03273..87c52d20b 100644
--- a/app/vmui/packages/vmui/src/pages/TopQueries/hooks/useSetQueryParams.ts
+++ b/app/vmui/packages/vmui/src/pages/TopQueries/hooks/useSetQueryParams.ts
@@ -1,10 +1,11 @@
 import { useTopQueriesState } from "../../../state/topQueries/TopQueriesStateContext";
 import { useEffect } from "react";
 import { compactObject } from "../../../utils/object";
-import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
+import { useSearchParams } from "react-router-dom";
 
 export const useSetQueryParams = () => {
   const { topN, maxLifetime } = useTopQueriesState();
+  const [, setSearchParams] = useSearchParams();
 
   const setSearchParamsFromState = () => {
     const params = compactObject({
@@ -12,7 +13,7 @@ export const useSetQueryParams = () => {
       maxLifetime: maxLifetime,
     });
 
-    setQueryStringWithoutPageReload(params);
+    setSearchParams(params);
   };
 
   useEffect(setSearchParamsFromState, [topN, maxLifetime]);
diff --git a/app/vmui/packages/vmui/src/pages/TracePage/index.tsx b/app/vmui/packages/vmui/src/pages/TracePage/index.tsx
index 498fccea6..b8307fa23 100644
--- a/app/vmui/packages/vmui/src/pages/TracePage/index.tsx
+++ b/app/vmui/packages/vmui/src/pages/TracePage/index.tsx
@@ -10,13 +10,14 @@ import { CloseIcon } from "../../components/Main/Icons";
 import Modal from "../../components/Main/Modal/Modal";
 import JsonForm from "./JsonForm/JsonForm";
 import { ErrorTypes } from "../../types";
-import { setQueryStringWithoutPageReload } from "../../utils/query-string";
+import { useSearchParams } from "react-router-dom";
 
 const TracePage: FC = () => {
   const [openModal, setOpenModal] = useState(false);
   const [tracesState, setTracesState] = useState<Trace[]>([]);
   const [errors, setErrors] = useState<{filename: string, text: string}[]>([]);
   const hasTraces = useMemo(() => !!tracesState.length, [tracesState]);
+  const [, setSearchParams] = useSearchParams();
 
   const handleOpenModal = () => {
     setOpenModal(true);
@@ -74,7 +75,7 @@ const TracePage: FC = () => {
   };
 
   useEffect(() => {
-    setQueryStringWithoutPageReload({});
+    setSearchParams({});
   }, []);
 
   const UploadButtons = () => (
diff --git a/app/vmui/packages/vmui/src/utils/query-string.ts b/app/vmui/packages/vmui/src/utils/query-string.ts
index 576a7be47..6a99e50da 100644
--- a/app/vmui/packages/vmui/src/utils/query-string.ts
+++ b/app/vmui/packages/vmui/src/utils/query-string.ts
@@ -2,27 +2,18 @@ import qs from "qs";
 import get from "lodash.get";
 import { MAX_QUERY_FIELDS } from "../constants/graph";
 
-export const setQueryStringWithoutPageReload = (params: Record<string, unknown>): void => {
-  const w = window;
-  if (w) {
-    const qsValue = Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent(String(v))}`).join("&");
-    const qs = qsValue ? `?${qsValue}` : "";
-    const newurl = `${w.location.protocol}//${w.location.host}${w.location.pathname}${qs}${w.location.hash}`;
-    w.history.pushState({ path: newurl }, "", newurl);
-  }
-};
-
 export const getQueryStringValue = (
   key: string,
   defaultValue?: unknown,
-  queryString = window.location.search
 ): unknown => {
+  const queryString = window.location.hash.split("?")[1];
   const values = qs.parse(queryString, { ignoreQueryPrefix: true });
   return get(values, key, defaultValue || "");
 };
 
 export const getQueryArray = (): string[] => {
-  const queryLength = window.location.search.match(/g\d+\.expr/g)?.length || 1;
+  const queryString = window.location.hash.split("?")[1] || "";
+  const queryLength = queryString.match(/g\d+\.expr/g)?.length || 1;
   return new Array(queryLength > MAX_QUERY_FIELDS ? MAX_QUERY_FIELDS : queryLength)
     .fill(1)
     .map((q, i) => getQueryStringValue(`g${i}.expr`, "") as string);
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3613c7b7c..3b7901dfe 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -19,11 +19,13 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * SECURITY: upgrade Go builder from Go1.20.0 to Go1.20.1. See [the list of issues addressed in Go1.20.1](https://github.com/golang/go/issues?q=milestone%3AGo1.20.1+label%3ACherryPickApproved).
 
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to VictoriaMetrics located in another datacenter or availability zone. This also allows reducing disk IO under high load when `vmagent` starts queuing the collected data to disk when the remote storage is temporarily unavailable or cannot keep up with the data ingestion rate. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [Kuma](http://kuma.io/) Control Plane targets discovery aka [kuma_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3389).
 * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway.html): add the ability to verify JWT signature via [JWKS endpoint](https://auth0.com/docs/secure/tokens/json-web-tokens/json-web-key-sets). See [these docs](https://docs.victoriametrics.com/vmgateway.html#using-jwks-endpoint-for-jwt-signature-verification).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add the ability to limit the number of concurrent requests on a per-user basis via `-maxConcurrentPerUserRequests` command-line flag and via `max_concurrent_requests` config option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346) and [these docs](https://docs.victoriametrics.com/vmauth.html#concurrency-limiting).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing `GET` requests on all [the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing). Previously the backend error has been immediately returned to the client without retrying the request on the remaining backends.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): choose the backend with the minimum number of concurrently executed requests [among the configured backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) in a round-robin manner for serving the incoming requests. This allows spreading the load among backends more evenly, while improving the response time.
 * FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): automatically retry requests to remote storage if up to 5 errors occur during the data migration process. This should help continuing the data migration process on temporary errors. Previously `vmctl` was stopping after the first error. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3600).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [share(q)](https://docs.victoriametrics.com/MetricsQL.html#share) aggregate function.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `mad_over_time(m[d])` function for calculating the [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) over raw samples on the lookbehind window `d`. See [this feature request](https://github.com/prometheus/prometheus/issues/5514).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `range_mad(q)` function for calculating the [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) over points per each time series returned by `q`.
@@ -32,10 +34,11 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `range_trim_zscore(z, q)` function for dropping outliers located farther than `z*range_stddev(q)` from `range_avg(q)`. This should help removing outliers during query time at [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3759).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show `median` instead of `avg` in graph tooltip and line legend, since `median` is more tolerant against spikes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3706).
 * FEATURE: allow setting zero value for `-search.latencyOffset` command-line flag. This may be needed in [some cases](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836). Previously the minimum supported value for `-search.latencyOffset` command-line flag was `1s`.
-* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `kuma_sd_config` for [Kuma](http://kuma.io/) Control Plane targets discovery. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3389).
 
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): immediately cancel in-flight scrape requests during configuration reload when [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode) is disabled. Previously `vmagent` could wait for long time until all the in-flight requests are completed before reloading the configuration. This could significantly slow down configuration reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3747).
 * BUGFIX: prevent from possible data ingestion slowdown and query performance slowdown during [background merges of big parts](https://docs.victoriametrics.com/#storage) on systems with small number of CPU cores (1 or 2 CPU cores). The issue has been introduced in [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850) when implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337). See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3790).
 * BUGFIX: properly parse timestamps in milliseconds when [ingesting data via OpenTSDB telnet put protocol](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol). Previously timestamps in milliseconds were mistakenly multiplied by 1000. Thanks to @Droxenator for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3810).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): do not add extrapolated points outside the real points when using [interpolate()](https://docs.victoriametrics.com/MetricsQL.html#interpolate) function. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3816).
 
 ## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1)
 
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index dd833969f..950866f46 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -222,7 +222,7 @@ the following approaches for automatic discovery of `vmstorage` nodes:
   The list of discovered `vmstorage` nodes is automatically updated when the file contents changes.
   The update frequency can be controlled with `-storageNode.discoveryInterval` command-line flag.
 
-- [dns+srv](https://en.wikipedia.org/wiki/SRV_record) - pass `dns+src:some-name` value to `-storageNode` command-line flag.
+- [dns+srv](https://en.wikipedia.org/wiki/SRV_record) - pass `dns+srv:some-name` value to `-storageNode` command-line flag.
   In this case the provided `dns+srv` names are resolved into tcp addresses of `vmstorage` nodes.
   The list of discovered `vmstorage` nodes is automatically updated at `vminsert` and `vmselect`
   when it changes behind the corresponding `dns+srv` names.
diff --git a/docs/README.md b/docs/README.md
index 27f3f0917..b5ced9de2 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -2339,6 +2339,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 7a3a2a046..551809482 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -109,6 +109,7 @@ Case studies:
 * [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Dig Security](https://docs.victoriametrics.com/CaseStudies.html#dig-security)
 * [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
 * [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
@@ -2341,6 +2342,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index 19dd2057c..6260f20cf 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -974,12 +974,10 @@ to one of the targets returned by the http service.
 
 The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling):
 
-* `__meta_server`: the URL of Kuma Control Plane's MADS xDS server from which the target was extracted
 * `__meta_kuma_mesh`: the name of the mesh
-* `__meta_kuma_service`: the name of the service associated with the proxy 
 * `__meta_kuma_dataplane`: the name of the proxy
+* `__meta_kuma_service`: the name of the service associated with the proxy
 * `__meta_kuma_label_<label_name>`: each label of target given from Kuma Control Plane
-* `__metrics_path__`: the path by which the service metrics are scraped
 
 ## nomad_sd_configs
 
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 9cd1419aa..b2d2a302f 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -1373,12 +1373,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#http_sd_configs for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
-  -promscrape.kumaSDCheckInterval duration
-     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 1m0s)
-  -promscrape.kuma.waitTime duration
-     Wait time used by Kuma service discovery. Default value is used if not set
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in Kuma Service Mesh API service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
diff --git a/lib/promscrape/discovery/docker/container.go b/lib/promscrape/discovery/docker/container.go
index f0b0aec2e..512ab212d 100644
--- a/lib/promscrape/discovery/docker/container.go
+++ b/lib/promscrape/discovery/docker/container.go
@@ -81,6 +81,7 @@ func addContainersLabels(containers []container, networkLabels map[string]*promu
 					m.Add("__meta_docker_port_public_ip", p.IP)
 				}
 				addCommonLabels(m, c, networkLabels[n.NetworkID])
+				// Remove possible duplicate labels, which can appear after addCommonLabels() call
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 				added = true
@@ -95,6 +96,7 @@ func addContainersLabels(containers []container, networkLabels map[string]*promu
 				m.Add("__address__", addr)
 				m.Add("__meta_docker_network_ip", n.IPAddress)
 				addCommonLabels(m, c, networkLabels[n.NetworkID])
+				// Remove possible duplicate labels, which can appear after addCommonLabels() call
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 			}
diff --git a/lib/promscrape/discovery/dockerswarm/services.go b/lib/promscrape/discovery/dockerswarm/services.go
index 725ceb0f4..88846671b 100644
--- a/lib/promscrape/discovery/dockerswarm/services.go
+++ b/lib/promscrape/discovery/dockerswarm/services.go
@@ -123,6 +123,7 @@ func addServicesLabels(services []service, networksLabels map[string]*promutils.
 				m.Add("__meta_dockerswarm_service_endpoint_port_publish_mode", ep.PublishMode)
 				m.AddFrom(commonLabels)
 				m.AddFrom(networksLabels[vip.NetworkID])
+				// Remove possible duplicate labels, which can appear after AddFrom() calls
 				m.RemoveDuplicates()
 				added = true
 				ms = append(ms, m)
@@ -132,6 +133,7 @@ func addServicesLabels(services []service, networksLabels map[string]*promutils.
 				m.Add("__address__", discoveryutils.JoinHostPort(ip.String(), port))
 				m.AddFrom(commonLabels)
 				m.AddFrom(networksLabels[vip.NetworkID])
+				// Remove possible duplicate labels, which can appear after AddFrom() calls
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 			}
diff --git a/lib/promscrape/discovery/dockerswarm/tasks.go b/lib/promscrape/discovery/dockerswarm/tasks.go
index 82cb009e4..6eec2e14d 100644
--- a/lib/promscrape/discovery/dockerswarm/tasks.go
+++ b/lib/promscrape/discovery/dockerswarm/tasks.go
@@ -111,6 +111,7 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []*promutils.Label
 			m.AddFrom(commonLabels)
 			m.Add("__address__", discoveryutils.JoinHostPort(commonLabels.Get("__meta_dockerswarm_node_address"), port.PublishedPort))
 			m.Add("__meta_dockerswarm_task_port_publish_mode", port.PublishMode)
+			// Remove possible duplicate labels, which can appear after AddFrom() call
 			m.RemoveDuplicates()
 			ms = append(ms, m)
 		}
@@ -132,6 +133,7 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []*promutils.Label
 					m.AddFrom(networkLabels)
 					m.Add("__address__", discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort))
 					m.Add("__meta_dockerswarm_task_port_publish_mode", ep.PublishMode)
+					// Remove possible duplicate labels, which can appear after AddFrom() calls
 					m.RemoveDuplicates()
 					ms = append(ms, m)
 					added = true
@@ -141,6 +143,7 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []*promutils.Label
 					m.AddFrom(commonLabels)
 					m.AddFrom(networkLabels)
 					m.Add("__address__", discoveryutils.JoinHostPort(ip.String(), port))
+					// Remove possible duplicate labels, which can appear after AddFrom() calls
 					m.RemoveDuplicates()
 					ms = append(ms, m)
 				}
diff --git a/lib/promscrape/discovery/http/http.go b/lib/promscrape/discovery/http/http.go
index a6eb180c7..23f4a5c01 100644
--- a/lib/promscrape/discovery/http/http.go
+++ b/lib/promscrape/discovery/http/http.go
@@ -47,6 +47,7 @@ func addHTTPTargetLabels(src []httpGroupTarget, sourceURL string) []*promutils.L
 			m.AddFrom(labels)
 			m.Add("__address__", target)
 			m.Add("__meta_url", sourceURL)
+			// Remove possible duplicate labels, which can appear after AddFrom() call
 			m.RemoveDuplicates()
 			ms = append(ms, m)
 		}
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index cff09bec7..f83526ed0 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -137,6 +137,7 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 				if svc != nil {
 					svc.appendCommonLabels(m)
 				}
+				// Remove possible duplicate labels, which can appear after appendCommonLabels() call
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 			}
@@ -155,6 +156,7 @@ func appendEndpointLabelsForAddresses(ms []*promutils.Labels, gw *groupWatcher,
 			}
 		}
 		m := getEndpointLabelsForAddressAndPort(gw, podPortsSeen, eps, ea, epp, p, svc, ready)
+		// Remove possible duplicate labels, which can appear inside getEndpointLabelsForAddressAndPort()
 		m.RemoveDuplicates()
 		ms = append(ms, m)
 	}
diff --git a/lib/promscrape/discovery/kubernetes/endpointslice.go b/lib/promscrape/discovery/kubernetes/endpointslice.go
index a7506b184..3d66c2264 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslice.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslice.go
@@ -56,6 +56,7 @@ func (eps *EndpointSlice) getTargetLabels(gw *groupWatcher) []*promutils.Labels
 		for _, epp := range eps.Ports {
 			for _, addr := range ess.Addresses {
 				m := getEndpointSliceLabelsForAddressAndPort(gw, podPortsSeen, addr, eps, ess, epp, p, svc)
+				// Remove possible duplicate labels, which can appear after getEndpointSliceLabelsForAddressAndPort() call
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 			}
@@ -86,6 +87,7 @@ func (eps *EndpointSlice) getTargetLabels(gw *groupWatcher) []*promutils.Labels
 				if svc != nil {
 					svc.appendCommonLabels(m)
 				}
+				// Remove possible duplicate labels, which can appear after appendCommonLabels() calls
 				m.RemoveDuplicates()
 				ms = append(ms, m)
 			}
diff --git a/lib/promscrape/discovery/kuma/api.go b/lib/promscrape/discovery/kuma/api.go
index 2e682a475..0e3560f43 100644
--- a/lib/promscrape/discovery/kuma/api.go
+++ b/lib/promscrape/discovery/kuma/api.go
@@ -4,30 +4,34 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"path"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var configMap = discoveryutils.NewConfigMap()
 
 type apiConfig struct {
-	client *discoveryutils.Client
-	path   string
+	client  *discoveryutils.Client
+	apiPath string
+
+	// labels contains the latest discovered labels.
+	labels atomic.Value
+
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
 
-	cancel        context.CancelFunc
-	wg            sync.WaitGroup
-	mu            sync.Mutex // protects targets
-	targets       []kumaTarget
 	latestVersion string
 	latestNonce   string
 
@@ -35,16 +39,6 @@ type apiConfig struct {
 	parseErrors *metrics.Counter
 }
 
-const (
-	discoveryNode      = "victoria-metrics"
-	xdsApiVersion      = "v3"
-	xdsRequestType     = "discovery"
-	xdsResourceType    = "monitoringassignments"
-	xdsResourceTypeUrl = "type.googleapis.com/kuma.observability.v1.MonitoringAssignment"
-)
-
-var waitTime = flag.Duration("promscrape.kuma.waitTime", 0, "Wait time used by Kuma service discovery. Default value is used if not set")
-
 func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
 	if err != nil {
@@ -54,16 +48,14 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 }
 
 func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
+	apiServer, apiPath, err := getAPIServerPath(sdc.Server)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse auth config: %w", err)
+		return nil, fmt.Errorf("cannot parse server %q: %w", sdc.Server, err)
 	}
-	parsedURL, err := url.Parse(sdc.Server)
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse kuma_sd server URL: %w", err)
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
-	apiServer := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
-
 	proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
@@ -73,135 +65,205 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}
 
-	apiPath := path.Join(
-		parsedURL.RequestURI(),
-		xdsApiVersion,
-		xdsRequestType+":"+xdsResourceType,
-	)
-
 	cfg := &apiConfig{
-		client:      client,
-		path:        apiPath,
+		client:  client,
+		apiPath: apiPath,
+
 		fetchErrors: metrics.GetOrCreateCounter(fmt.Sprintf(`promscrape_discovery_kuma_errors_total{type="fetch",url=%q}`, sdc.Server)),
 		parseErrors: metrics.GetOrCreateCounter(fmt.Sprintf(`promscrape_discovery_kuma_errors_total{type="parse",url=%q}`, sdc.Server)),
 	}
 
-	// initialize targets synchronously and start updating them in background
-	cfg.startWatcher()
-
-	return cfg, nil
-}
-
-func (cfg *apiConfig) startWatcher() func() {
 	ctx, cancel := context.WithCancel(context.Background())
 	cfg.cancel = cancel
 
-	// blocking initial targets update
-	if err := cfg.updateTargets(ctx); err != nil {
-		logger.Errorf("there were errors when discovering kuma targets, so preserving the previous targets. error: %v", err)
+	// Initialize targets synchronously and then start updating them in background.
+	// The synchronous targets' update is needed for returning non-empty list of targets
+	// just after the initialization.
+	if err := cfg.updateTargetsLabels(ctx); err != nil {
+		return nil, fmt.Errorf("cannot discover Kuma targets: %w", err)
 	}
-
-	// start updating targets with a long polling in background
 	cfg.wg.Add(1)
 	go func() {
-		ticker := time.NewTicker(*SDCheckInterval)
 		defer cfg.wg.Done()
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ticker.C:
-				// we are constantly waiting for targets updates in long polling requests
-				err := cfg.updateTargets(ctx)
-				if err != nil {
-					logger.Errorf("there were errors when discovering kuma targets, so preserving the previous targets. error: %v", err)
-				}
-			case <-ctx.Done():
-				return
-			}
-		}
+		cfg.runTargetsWatcher(ctx)
 	}()
 
-	return cancel
+	return cfg, nil
 }
 
-func (cfg *apiConfig) stopWatcher() {
-	cfg.cancel()
-	cfg.wg.Wait()
+func getAPIServerPath(serverURL string) (string, string, error) {
+	if serverURL == "" {
+		return "", "", fmt.Errorf("missing servier url")
+	}
+	if !strings.Contains(serverURL, "://") {
+		serverURL = "http://" + serverURL
+	}
+	psu, err := url.Parse(serverURL)
+	if err != nil {
+		return "", "", fmt.Errorf("cannot parse server url=%q: %w", serverURL, err)
+	}
+	apiServer := fmt.Sprintf("%s://%s", psu.Scheme, psu.Host)
+	apiPath := psu.Path
+	if !strings.HasSuffix(apiPath, "/") {
+		apiPath += "/"
+	}
+	apiPath += "v3/discovery:monitoringassignments"
+	if psu.RawQuery != "" {
+		apiPath += "?" + psu.RawQuery
+	}
+	return apiServer, apiPath, nil
 }
 
-func (cfg *apiConfig) getTargets() ([]kumaTarget, error) {
-	cfg.mu.Lock()
-	defer cfg.mu.Unlock()
-	return cfg.targets, nil
-}
+func (cfg *apiConfig) runTargetsWatcher(ctx context.Context) {
+	ticker := time.NewTicker(*SDCheckInterval)
+	defer ticker.Stop()
 
-func (cfg *apiConfig) updateTargets(ctx context.Context) error {
-	requestBody, err := json.Marshal(discoveryRequest{
-		VersionInfo:   cfg.latestVersion,
-		Node:          discoveryRequestNode{Id: discoveryNode},
-		TypeUrl:       xdsResourceTypeUrl,
-		ResponseNonce: cfg.latestNonce,
-	})
-	if err != nil {
-		return fmt.Errorf("cannot marshal request body for kuma_sd api: %w", err)
+	doneCh := ctx.Done()
+	for {
+		select {
+		case <-ticker.C:
+			if err := cfg.updateTargetsLabels(ctx); err != nil {
+				logger.Errorf("there was an error when discovering Kuma targets, so preserving the previous targets; error: %s", err)
+			}
+		case <-doneCh:
+			return
+		}
 	}
+}
 
-	var statusCode int
-	data, err := cfg.client.GetBlockingAPIResponseWithParamsCtx(
-		ctx,
-		cfg.path,
-		func(request *http.Request) {
-			request.Method = http.MethodPost
-			request.Body = io.NopCloser(bytes.NewReader(requestBody))
-
-			// set max duration for long polling request
-			query := request.URL.Query()
-			query.Add("fetch-timeout", cfg.getWaitTime().String())
-			request.URL.RawQuery = query.Encode()
+func (cfg *apiConfig) mustStop() {
+	cfg.client.Stop()
+	cfg.cancel()
+	cfg.wg.Wait()
+}
 
-			request.Header.Set("Accept", "application/json")
-			request.Header.Set("Content-Type", "application/json")
+func (cfg *apiConfig) updateTargetsLabels(ctx context.Context) error {
+	dReq := &discoveryRequest{
+		VersionInfo: cfg.latestVersion,
+		Node: discoveryRequestNode{
+			ID: "vmagent",
 		},
-		func(response *http.Response) {
-			statusCode = response.StatusCode
-		},
-	)
-
-	if statusCode == http.StatusNotModified {
-		return nil
+		TypeURL:       "type.googleapis.com/kuma.observability.v1.MonitoringAssignment",
+		ResponseNonce: cfg.latestNonce,
+	}
+	requestBody, err := json.Marshal(dReq)
+	if err != nil {
+		logger.Panicf("BUG: cannot marshal Kuma discovery request: %s", err)
 	}
+	updateRequestFunc := func(req *http.Request) {
+		req.Method = "POST"
+		req.Body = io.NopCloser(bytes.NewReader(requestBody))
+		req.Header.Set("Accept", "application/json")
+		req.Header.Set("Content-Type", "application/json")
+	}
+	notModified := false
+	inspectResponseFunc := func(resp *http.Response) {
+		if resp.StatusCode == http.StatusNotModified {
+			// Override status code, so GetBlockingAPIResponseWithParamsCtx() returns nil error.
+			resp.StatusCode = http.StatusOK
+			notModified = true
+		}
+	}
+	data, err := cfg.client.GetBlockingAPIResponseWithParamsCtx(ctx, cfg.apiPath, updateRequestFunc, inspectResponseFunc)
 	if err != nil {
 		cfg.fetchErrors.Inc()
-		return fmt.Errorf("cannot read kuma_sd api response: %w", err)
+		return fmt.Errorf("error when reading Kuma discovery response: %w", err)
+	}
+	if notModified {
+		// The targets weren't modified, so nothing to update.
+		return nil
 	}
 
-	response, err := parseDiscoveryResponse(data)
+	// Parse response
+	labels, versionInfo, nonce, err := parseTargetsLabels(data)
 	if err != nil {
 		cfg.parseErrors.Inc()
-		return fmt.Errorf("cannot parse kuma_sd api response: %w", err)
+		return fmt.Errorf("cannot parse Kuma discovery response received from %q: %w", cfg.client.APIServer(), err)
 	}
-
-	cfg.mu.Lock()
-	defer cfg.mu.Unlock()
-	cfg.targets = parseKumaTargets(response)
-	cfg.latestVersion = response.VersionInfo
-	cfg.latestNonce = response.Nonce
+	cfg.labels.Store(&labels)
+	cfg.latestVersion = versionInfo
+	cfg.latestNonce = nonce
 
 	return nil
 }
 
-func (cfg *apiConfig) getWaitTime() time.Duration {
-	d := discoveryutils.BlockingClientReadTimeout
-	// Reduce wait time to avoid timeouts (request execution time should be less than the read timeout)
-	d -= d / 8
-	if *waitTime > time.Second && *waitTime < d {
-		d = *waitTime
+func parseTargetsLabels(data []byte) ([]*promutils.Labels, string, string, error) {
+	var dResp discoveryResponse
+	if err := json.Unmarshal(data, &dResp); err != nil {
+		return nil, "", "", err
 	}
-	return d
+	return dResp.getTargetsLabels(), dResp.VersionInfo, dResp.Nonce, nil
 }
 
-func (cfg *apiConfig) mustStop() {
-	cfg.stopWatcher()
-	cfg.client.Stop()
+func (dr *discoveryResponse) getTargetsLabels() []*promutils.Labels {
+	var ms []*promutils.Labels
+	for _, r := range dr.Resources {
+		for _, t := range r.Targets {
+			m := promutils.NewLabels(8 + len(r.Labels) + len(t.Labels))
+
+			m.Add("instance", t.Name)
+			m.Add("__address__", t.Address)
+			m.Add("__scheme__", t.Scheme)
+			m.Add("__metrics_path__", t.MetricsPath)
+			m.Add("__meta_kuma_dataplane", t.Name)
+			m.Add("__meta_kuma_mesh", r.Mesh)
+			m.Add("__meta_kuma_service", r.Service)
+
+			addLabels(m, r.Labels)
+			addLabels(m, t.Labels)
+			// Remove possible duplicate labels after addLabels() calls above
+			m.RemoveDuplicates()
+
+			ms = append(ms, m)
+		}
+	}
+	return ms
+}
+
+func addLabels(dst *promutils.Labels, src map[string]string) {
+	bb := bbPool.Get()
+	b := bb.B
+	for k, v := range src {
+		b = append(b[:0], "__meta_kuma_label_"...)
+		b = append(b, discoveryutils.SanitizeLabelName(k)...)
+		labelName := bytesutil.InternBytes(b)
+		dst.Add(labelName, v)
+	}
+	bb.B = b
+	bbPool.Put(bb)
+}
+
+var bbPool bytesutil.ByteBufferPool
+
+// discoveryRequest represent xDS-requests for Kuma Service Mesh
+// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/discovery/v3/discovery.proto#envoy-v3-api-msg-service-discovery-v3-discoveryrequest
+type discoveryRequest struct {
+	VersionInfo   string               `json:"version_info"`
+	Node          discoveryRequestNode `json:"node"`
+	ResourceNames []string             `json:"resource_names"`
+	TypeURL       string               `json:"type_url"`
+	ResponseNonce string               `json:"response_nonce"`
+}
+
+type discoveryRequestNode struct {
+	ID string `json:"id"`
+}
+
+// discoveryResponse represent xDS-requests for Kuma Service Mesh
+// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/discovery/v3/discovery.proto#envoy-v3-api-msg-service-discovery-v3-discoveryresponse
+type discoveryResponse struct {
+	VersionInfo string `json:"version_info"`
+	Resources   []struct {
+		Mesh    string `json:"mesh"`
+		Service string `json:"service"`
+		Targets []struct {
+			Name        string            `json:"name"`
+			Scheme      string            `json:"scheme"`
+			Address     string            `json:"address"`
+			MetricsPath string            `json:"metrics_path"`
+			Labels      map[string]string `json:"labels"`
+		} `json:"targets"`
+		Labels map[string]string `json:"labels"`
+	} `json:"resources"`
+	Nonce string `json:"nonce"`
 }
diff --git a/lib/promscrape/discovery/kuma/kuma.go b/lib/promscrape/discovery/kuma/kuma.go
index a3c62fc61..7d42a46ba 100644
--- a/lib/promscrape/discovery/kuma/kuma.go
+++ b/lib/promscrape/discovery/kuma/kuma.go
@@ -11,7 +11,7 @@ import (
 )
 
 // SDCheckInterval defines interval for targets refresh.
-var SDCheckInterval = flag.Duration("promscrape.kumaSDCheckInterval", time.Minute, "Interval for checking for changes in kuma service discovery. "+
+var SDCheckInterval = flag.Duration("promscrape.kumaSDCheckInterval", 30*time.Second, "Interval for checking for changes in kuma service discovery. "+
 	"This works only if kuma_sd_configs is configured in '-promscrape.config' file. "+
 	"See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details")
 
@@ -19,22 +19,17 @@ var SDCheckInterval = flag.Duration("promscrape.kumaSDCheckInterval", time.Minut
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kuma_sd_config
 type SDConfig struct {
-	Server            string                     `yaml:"server"`
+	Server string `yaml:"server"`
+
 	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
 	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`
 	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
-}
 
-type kumaTarget struct {
-	Mesh         string            `json:"mesh"`
-	ControlPlane string            `json:"controlplane"`
-	Service      string            `json:"service"`
-	DataPlane    string            `json:"dataplane"`
-	Instance     string            `json:"instance"`
-	Scheme       string            `json:"scheme"`
-	Address      string            `json:"address"`
-	MetricsPath  string            `json:"metrics_path"`
-	Labels       map[string]string `json:"labels"`
+	// fetch_timeout isn't used, so it isn't defined.
+	// FetchTimeout time.Duration `yaml:"fetch_timeout,omitempty"`
+
+	// refresh_interval is obtained from `-promscrape.kumaSDCheckInterval` command-line option.
+	// RefreshInterval time.Duration `yaml:"refresh_interval,omitempty"`
 }
 
 // GetLabels returns kuma service discovery labels according to sdc.
@@ -43,11 +38,9 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot get API config for kuma_sd: %w", err)
 	}
-	targets, err := cfg.getTargets()
-	if err != nil {
-		return nil, err
-	}
-	return kumaTargetsToLabels(targets, sdc.Server), nil
+	v := cfg.labels.Load()
+	pLabels := v.(*[]*promutils.Labels)
+	return *pLabels, nil
 }
 
 // MustStop stops further usage for sdc.
@@ -59,55 +52,3 @@ func (sdc *SDConfig) MustStop() {
 		cfg.mustStop()
 	}
 }
-
-func kumaTargetsToLabels(src []kumaTarget, sourceURL string) []*promutils.Labels {
-	ms := make([]*promutils.Labels, 0, len(src))
-	for _, target := range src {
-		m := promutils.NewLabels(8 + len(target.Labels))
-
-		m.Add("instance", target.Instance)
-		m.Add("__address__", target.Address)
-		m.Add("__scheme__", target.Scheme)
-		m.Add("__metrics_path__", target.MetricsPath)
-		m.Add("__meta_server", sourceURL)
-		m.Add("__meta_kuma_mesh", target.Mesh)
-		m.Add("__meta_kuma_service", target.Service)
-		m.Add("__meta_kuma_dataplane", target.DataPlane)
-		for k, v := range target.Labels {
-			m.Add("__meta_kuma_label_"+k, v)
-		}
-
-		m.RemoveDuplicates()
-		ms = append(ms, m)
-	}
-	return ms
-}
-
-func parseKumaTargets(response discoveryResponse) []kumaTarget {
-	result := make([]kumaTarget, 0, len(response.Resources))
-
-	for _, resource := range response.Resources {
-		for _, target := range resource.Targets {
-			labels := make(map[string]string)
-			for label, value := range resource.Labels {
-				labels[label] = value
-			}
-			for label, value := range target.Labels {
-				labels[label] = value
-			}
-			result = append(result, kumaTarget{
-				Mesh:         resource.Mesh,
-				ControlPlane: response.ControlPlane.Identifier,
-				Service:      resource.Service,
-				DataPlane:    target.Name,
-				Instance:     target.Name,
-				Scheme:       target.Scheme,
-				Address:      target.Address,
-				MetricsPath:  target.MetricsPath,
-				Labels:       labels,
-			})
-		}
-	}
-
-	return result
-}
diff --git a/lib/promscrape/discovery/kuma/xds.go b/lib/promscrape/discovery/kuma/xds.go
deleted file mode 100644
index 32fcdafa4..000000000
--- a/lib/promscrape/discovery/kuma/xds.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package kuma
-
-import (
-	"encoding/json"
-	"fmt"
-)
-
-// discoveryRequest represent xDS-requests for Kuma Service Mesh
-// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/discovery/v3/discovery.proto#envoy-v3-api-msg-service-discovery-v3-discoveryrequest
-type discoveryRequest struct {
-	VersionInfo   string               `json:"version_info,omitempty"`
-	Node          discoveryRequestNode `json:"node,omitempty"`
-	ResourceNames []string             `json:"resource_names,omitempty"`
-	TypeUrl       string               `json:"type_url,omitempty"`
-	ResponseNonce string               `json:"response_nonce,omitempty"`
-}
-
-type discoveryRequestNode struct {
-	Id string `json:"id,omitempty"`
-}
-
-// discoveryResponse represent xDS-requests for Kuma Service Mesh
-// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/discovery/v3/discovery.proto#envoy-v3-api-msg-service-discovery-v3-discoveryresponse
-type discoveryResponse struct {
-	VersionInfo string `json:"version_info,omitempty"`
-	Resources   []struct {
-		Mesh    string `json:"mesh,omitempty"`
-		Service string `json:"service,omitempty"`
-		Targets []struct {
-			Name        string            `json:"name,omitempty"`
-			Scheme      string            `json:"scheme,omitempty"`
-			Address     string            `json:"address,omitempty"`
-			MetricsPath string            `json:"metrics_path,omitempty"`
-			Labels      map[string]string `json:"labels,omitempty"`
-		} `json:"targets,omitempty"`
-		Labels map[string]string `json:"labels,omitempty"`
-	} `json:"resources,omitempty"`
-	TypeUrl      string `json:"type_url,omitempty"`
-	Nonce        string `json:"nonce,omitempty"`
-	ControlPlane struct {
-		Identifier string `json:"identifier,omitempty"`
-	} `json:"control_plane,omitempty"`
-}
-
-func parseDiscoveryResponse(data []byte) (discoveryResponse, error) {
-	response := discoveryResponse{}
-	err := json.Unmarshal(data, &response)
-	if err != nil {
-		return discoveryResponse{}, fmt.Errorf("cannot parse kuma_sd api response, err:  %w", err)
-	}
-	if response.TypeUrl != xdsResourceTypeUrl {
-		return discoveryResponse{}, fmt.Errorf("unexpected type_url in kuma_sd api response, expected: %s, got: %s", xdsResourceTypeUrl, response.TypeUrl)
-	}
-
-	return response, nil
-}
diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go
index 714c4b16f..50f5ea0db 100644
--- a/lib/promscrape/discoveryutils/client.go
+++ b/lib/promscrape/discoveryutils/client.go
@@ -42,8 +42,11 @@ const (
 	DefaultClientReadTimeout = time.Minute
 )
 
-type RequestCallback func(request *http.Request)
-type ResponseCallback func(response *http.Response)
+// RequestCallback is called on the request before sending the request to the server.
+type RequestCallback func(req *http.Request)
+
+// ResponseCallback is called on the response before validating and returning the response to the caller.
+type ResponseCallback func(resp *http.Response)
 
 func concurrencyLimitChInit() {
 	concurrencyLimitCh = make(chan struct{}, *maxConcurrency)


Test patch:
diff --git a/app/vmagent/remotewrite/pendingseries_test.go b/app/vmagent/remotewrite/pendingseries_test.go
index ef4ab11ee..5b7a51219 100644
--- a/app/vmagent/remotewrite/pendingseries_test.go
+++ b/app/vmagent/remotewrite/pendingseries_test.go
@@ -33,16 +33,6 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 			pushBlockLen = len(block)
 		}
 		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
-
-		if pushBlockLen == expectedBlockLen {
-			return
-		}
-
-		if tolerancePrc == 0 {
-			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes",
-				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen)
-		}
-
 		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
 			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
 				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index c8ff5e8d1..8706a363a 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -6212,7 +6212,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() < 1300)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1200, 1200, 1200, 1200},
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -6223,7 +6223,18 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() > 1500)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1800, 2000},
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`interpolate(tail_head_and_middle)`, func(t *testing.T) {
+		t.Parallel()
+		q := `interpolate(time() > 1100 and time() < 1300 default time() > 1700 and time() < 1900)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1200, 1400, 1600, 1800, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
diff --git a/lib/promscrape/discovery/kuma/api_test.go b/lib/promscrape/discovery/kuma/api_test.go
index ee11ac69a..ac2229540 100644
--- a/lib/promscrape/discovery/kuma/api_test.go
+++ b/lib/promscrape/discovery/kuma/api_test.go
@@ -1,80 +1,61 @@
 package kuma
 
 import (
-	"reflect"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-func Test_buildAPIPath(t *testing.T) {
-	type args struct {
-		server string
+func TestGetAPIServerPathSuccess(t *testing.T) {
+	f := func(server, expectedAPIServer, expectedAPIPath string) {
+		t.Helper()
+		apiServer, apiPath, err := getAPIServerPath(server)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if apiServer != expectedAPIServer {
+			t.Fatalf("unexpected API server; got %q; want %q", apiServer, expectedAPIServer)
+		}
+		if apiPath != expectedAPIPath {
+			t.Fatalf("unexpected API path; got %q; want %q", apiPath, expectedAPIPath)
+		}
 	}
-	tests := []struct {
-		name    string
-		args    args
-		want    string
-		wantErr bool
-	}{
-		{
-			name: "get api path ok",
-			args: args{server: "http://localhost:5676"},
-			want: "/v3/discovery:monitoringassignments",
-		},
-		{
-			name:    "get api path incorrect server URL",
-			args:    args{server: ":"},
-			wantErr: true,
-		},
-		{
-			name:    "get api path incorrect server URL err",
-			args:    args{server: "api"},
-			wantErr: true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			sdConf := &SDConfig{
-				Server:            tt.args.server,
-				HTTPClientConfig:  promauth.HTTPClientConfig{},
-				ProxyClientConfig: promauth.ProxyClientConfig{},
-			}
-			apiConf, err := getAPIConfig(sdConf, ".")
-
-			if tt.wantErr {
-				if err == nil {
-					t.Errorf("buildAPIPath() error = %v, wantErr %v", err, tt.wantErr)
-				}
-				return
-			}
-
-			if apiConf.path != tt.want {
-				t.Errorf("buildAPIPath() got = %v, want = %v", apiConf.path, tt.want)
-			}
-
-			sdConf.MustStop()
-		})
-	}
-
+	// url without path
+	f("http://localhost:5676", "http://localhost:5676", "/v3/discovery:monitoringassignments")
+	// url with path
+	f("http://localhost:5676/", "http://localhost:5676", "/v3/discovery:monitoringassignments")
+	f("https://foo.bar:1234/a/b", "https://foo.bar:1234", "/a/b/v3/discovery:monitoringassignments")
+	// url with query args
+	f("https://foo.bar:1234/a/b?c=d&arg2=value2", "https://foo.bar:1234", "/a/b/v3/discovery:monitoringassignments?c=d&arg2=value2")
+	// missing scheme
+	f("foo.bar", "http://foo.bar", "/v3/discovery:monitoringassignments")
+	f("foo.bar:1234/a/b", "http://foo.bar:1234", "/a/b/v3/discovery:monitoringassignments")
+	f("foo.bar:1234/a/b?c=d&arg2=value2", "http://foo.bar:1234", "/a/b/v3/discovery:monitoringassignments?c=d&arg2=value2")
 }
 
-func Test_parseAPIResponse(t *testing.T) {
-	type args struct {
-		data []byte
+func TestGetAPIConfigFailure(t *testing.T) {
+	f := func(server string) {
+		t.Helper()
+		sdc := &SDConfig{
+			Server: server,
+		}
+		cfg, err := getAPIConfig(sdc, ".")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if cfg != nil {
+			t.Fatalf("expecting nil cfg; got %v", cfg)
+		}
 	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []kumaTarget
-		wantErr bool
-	}{
+	// empty server url
+	f("")
+	// invalid server url
+	f(":")
+}
 
-		{
-			name: "parse ok",
-			args: args{
-				data: []byte(`{
+func TestParseTargetsLabels(t *testing.T) {
+	data := `{
     "version_info":"5dc9a5dd-2091-4426-a886-dfdc24fc99d7",
     "resources":[
        {
@@ -100,77 +81,55 @@ func Test_parseAPIResponse(t *testing.T) {
           "targets":[
              {
                 "name":"app",
-                "scheme":"http",
+                "scheme":"https",
                 "address":"127.0.0.1:5671",
-                "metrics_path":"/metrics",
+                "metrics_path":"/metrics/abc",
                 "labels":{ "kuma_io_protocol":"http" }
              }
           ]
        }
     ],
-    "type_url":"type.googleapis.com/kuma.observability.v1.MonitoringAssignment"
- }`),
-			},
-			want: []kumaTarget{
-				{
-					Mesh:        "default",
-					Service:     "redis",
-					DataPlane:   "redis",
-					Instance:    "redis",
-					Scheme:      "http",
-					Address:     "127.0.0.1:5670",
-					MetricsPath: "/metrics",
-					Labels:      map[string]string{"kuma_io_protocol": "tcp", "test": "test1"},
-				},
-				{
-					Mesh:        "default",
-					Service:     "app",
-					DataPlane:   "app",
-					Instance:    "app",
-					Scheme:      "http",
-					Address:     "127.0.0.1:5671",
-					MetricsPath: "/metrics",
-					Labels:      map[string]string{"kuma_io_protocol": "http", "test": "test2"},
-				},
-			},
-		},
-		{
-			name:    "parse err",
-			args:    args{data: []byte(`[]`)},
-			wantErr: true,
-		},
-		{
-			name: "api version err",
-			args: args{
-				data: []byte(`{
-    "resources":[
-       {
-          "@type":"type.googleapis.com/kuma.observability.v2.MonitoringAssignment",
-          "mesh":"default",
-          "service":"redis",
-          "targets":[]
-       }
-    ],
-    "type_url":"type.googleapis.com/kuma.observability.v2.MonitoringAssignment"
- }`),
-			},
-			wantErr: true,
-		},
+    "type_url":"type.googleapis.com/kuma.observability.v1.MonitoringAssignment",
+    "nonce": "foobar"
+ }`
+	labelss, versionInfo, nonce, err := parseTargetsLabels([]byte(data))
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	expectedLabelss := []*promutils.Labels{
+		promutils.NewLabelsFromMap(map[string]string{
+			"__address__":                        "127.0.0.1:5670",
+			"__meta_kuma_dataplane":              "redis",
+			"__meta_kuma_label_kuma_io_protocol": "tcp",
+			"__meta_kuma_label_test":             "test1",
+			"__meta_kuma_mesh":                   "default",
+			"__meta_kuma_service":                "redis",
+			"__metrics_path__":                   "/metrics",
+			"__scheme__":                         "http",
+			"instance":                           "redis",
+		}),
+		promutils.NewLabelsFromMap(map[string]string{
+			"__address__":                        "127.0.0.1:5671",
+			"__meta_kuma_dataplane":              "app",
+			"__meta_kuma_label_kuma_io_protocol": "http",
+			"__meta_kuma_label_test":             "test2",
+			"__meta_kuma_mesh":                   "default",
+			"__meta_kuma_service":                "app",
+			"__metrics_path__":                   "/metrics/abc",
+			"__scheme__":                         "https",
+			"instance":                           "app",
+		}),
+	}
+	discoveryutils.TestEqualLabelss(t, labelss, expectedLabelss)
+
+	expectedVersionInfo := "5dc9a5dd-2091-4426-a886-dfdc24fc99d7"
+	if versionInfo != expectedVersionInfo {
+		t.Fatalf("unexpected versionInfo; got %q; want %q", versionInfo, expectedVersionInfo)
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			resp, err := parseDiscoveryResponse(tt.args.data)
-			if tt.wantErr {
-				if err == nil {
-					t.Errorf("parseDiscoveryResponse() error = %v, wantErr %v", err, tt.wantErr)
-				}
-				return
-			}
 
-			got := parseKumaTargets(resp)
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseDiscoveryResponse() got = %v, want %v", got, tt.want)
-			}
-		})
+	expectedNonce := "foobar"
+	if nonce != expectedNonce {
+		t.Fatalf("unexpected nonce; got %q; want %q", nonce, expectedNonce)
 	}
 }
diff --git a/lib/promscrape/discovery/kuma/kuma_test.go b/lib/promscrape/discovery/kuma/kuma_test.go
deleted file mode 100644
index 683e4b545..000000000
--- a/lib/promscrape/discovery/kuma/kuma_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package kuma
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-func Test_kumaTargetsToLabels(t *testing.T) {
-	type args struct {
-		src []kumaTarget
-	}
-	tests := []struct {
-		name string
-		args args
-		want []*promutils.Labels
-	}{
-		{
-			name: "convert to labels ok",
-			args: args{
-				src: []kumaTarget{
-					{
-						Mesh:        "default",
-						Service:     "redis",
-						DataPlane:   "redis",
-						Instance:    "redis",
-						Scheme:      "http",
-						Address:     "127.0.0.1:5670",
-						MetricsPath: "/metrics",
-						Labels:      map[string]string{"kuma_io_protocol": "tcp", "kuma_io_service": "redis"},
-					},
-					{
-						Mesh:        "default",
-						Service:     "app",
-						DataPlane:   "app",
-						Instance:    "app",
-						Scheme:      "http",
-						Address:     "127.0.0.1:5671",
-						MetricsPath: "/vm/metrics",
-						Labels:      map[string]string{"kuma_io_protocol": "http", "kuma_io_service": "app"},
-					},
-				},
-			},
-			want: []*promutils.Labels{
-				promutils.NewLabelsFromMap(map[string]string{
-					"instance":                           "redis",
-					"__address__":                        "127.0.0.1:5670",
-					"__scheme__":                         "http",
-					"__metrics_path__":                   "/metrics",
-					"__meta_server":                      "http://localhost:5676",
-					"__meta_kuma_mesh":                   "default",
-					"__meta_kuma_service":                "redis",
-					"__meta_kuma_dataplane":              "redis",
-					"__meta_kuma_label_kuma_io_protocol": "tcp",
-					"__meta_kuma_label_kuma_io_service":  "redis",
-				}),
-				promutils.NewLabelsFromMap(map[string]string{
-					"instance":                           "app",
-					"__address__":                        "127.0.0.1:5671",
-					"__scheme__":                         "http",
-					"__metrics_path__":                   "/vm/metrics",
-					"__meta_server":                      "http://localhost:5676",
-					"__meta_kuma_mesh":                   "default",
-					"__meta_kuma_service":                "app",
-					"__meta_kuma_dataplane":              "app",
-					"__meta_kuma_label_kuma_io_protocol": "http",
-					"__meta_kuma_label_kuma_io_service":  "app",
-				}),
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := kumaTargetsToLabels(tt.args.src, "http://localhost:5676")
-			discoveryutils.TestEqualLabelss(t, got, tt.want)
-		})
-	}
-}