Repository: VictoriaMetrics/VictoriaMetrics
Pull request #6594 (2024-07-05 13:00:47)

docs/vmanomaly:custom model guide fix
### Describe Your Changes

Fixed Custom Model guide according to newer `vmanomaly` versions

### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).



Patch:
diff --git a/README.md b/README.md
index 0277424fa..21fc503da 100644
--- a/README.md
+++ b/README.md
@@ -3088,7 +3088,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxLabelsAPIDuration duration
      The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. See also -search.maxLabelsAPISeries and -search.ignoreExtraFiltersAtLabelsAPI (default 5s)
   -search.maxLabelsAPISeries int
-     The maximum number of time series, which could be scanned when searching for the the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
+     The maximum number of time series, which could be scanned when searching for the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
   -search.maxMemoryPerQuery size
diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index 61047f836..cd9c10977 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -52,7 +52,7 @@ var (
 	maxExportSeries     = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
 	maxTSDBStatusSeries = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
 	maxSeriesLimit      = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
-	maxLabelsAPISeries  = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the the matching time series "+
+	maxLabelsAPISeries  = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the matching time series "+
 		"at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, "+
 		"-search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI")
 	maxPointsPerTimeseries = flag.Int("search.maxPointsPerTimeseries", 30e3, "The maximum points per a single timeseries returned from /api/v1/query_range. "+
diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json
index 80d5ee771..27ad605a1 100644
--- a/dashboards/vm/vmagent.json
+++ b/dashboards/vm/vmagent.json
@@ -4978,7 +4978,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5075,7 +5075,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5178,7 +5178,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_flushed_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_flushed_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; output={{output}}",
               "range": true,
@@ -5276,7 +5276,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])",
+              "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "instant": false,
               "legendFormat": "aggregation: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5288,7 +5288,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])",
+              "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "hide": false,
               "instant": false,
               "legendFormat": "deduplication: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
@@ -5388,7 +5388,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "histogram_quantile(0.99, rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))",
+              "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json
index fffa75163..3cf857e91 100644
--- a/dashboards/vmagent.json
+++ b/dashboards/vmagent.json
@@ -4977,7 +4977,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5074,7 +5074,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5177,7 +5177,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "rate(vm_streamaggr_flushed_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
+              "expr": "sum(rate(vm_streamaggr_flushed_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; output={{output}}",
               "range": true,
@@ -5275,7 +5275,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])",
+              "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "instant": false,
               "legendFormat": "aggregation: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
@@ -5287,7 +5287,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])",
+              "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "hide": false,
               "instant": false,
               "legendFormat": "deduplication: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
@@ -5387,7 +5387,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "histogram_quantile(0.99, rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))",
+              "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))",
               "instant": false,
               "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
               "range": true,
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index e05391968..78df8aaad 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -1523,7 +1523,7 @@ Below is the output for `/path/to/vmselect -help`:
   -search.maxLabelsAPIDuration duration
      The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. See also -search.maxLabelsAPISeries and -search.ignoreExtraFiltersAtLabelsAPI (default 5s)
   -search.maxLabelsAPISeries int
-     The maximum number of time series, which could be scanned when searching for the the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
+     The maximum number of time series, which could be scanned when searching for the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
   -search.maxMemoryPerQuery size
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 845c4df91..66a6d900f 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -3099,7 +3099,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxLabelsAPIDuration duration
      The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. See also -search.maxLabelsAPISeries and -search.ignoreExtraFiltersAtLabelsAPI (default 5s)
   -search.maxLabelsAPISeries int
-     The maximum number of time series, which could be scanned when searching for the the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
+     The maximum number of time series, which could be scanned when searching for the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
   -search.maxMemoryPerQuery size
diff --git a/docs/VictoriaLogs/victorialogs-datasource.md b/docs/VictoriaLogs/victorialogs-datasource.md
index beccd1627..27d15cf6b 100644
--- a/docs/VictoriaLogs/victorialogs-datasource.md
+++ b/docs/VictoriaLogs/victorialogs-datasource.md
@@ -1,7 +1,7 @@
 ---
 weight: 9
 title: VictoriaLogs datasource for Grafana
-source: https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/README.md
+editLink: https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/README.md
 menu:
   docs:
     parent: victorialogs
@@ -135,9 +135,9 @@ extraInitContainers:
        set -ex
        mkdir -p /var/lib/grafana/plugins/
        ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-       curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/plugin.tar.gz
-       tar -xf /var/lib/grafana/plugins/plugin.tar.gz -C /var/lib/grafana/plugins/
-       rm /var/lib/grafana/plugins/plugin.tar.gz
+       curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
+       tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
+       rm /var/lib/grafana/plugins/vl-plugin.tar.gz
     volumeMounts:
       # For grafana-operator users, change `name: storage` to `name: grafana-data`
       - name: storage
@@ -186,18 +186,18 @@ spec:
               command: [ "/bin/sh" ]
               workingDir: "/var/lib/grafana"
               securityContext:
-                runAsUser: 10001
+                runAsUser: 472 
                 runAsNonRoot: true
-                runAsGroup: 10001
+                runAsGroup: 472 
               args:
                 - "-c"
                 - |
                   set -ex
                   mkdir -p /var/lib/grafana/plugins/
                   ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-                  curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/plugin.tar.gz
-                  tar -xf /var/lib/grafana/plugins/plugin.tar.gz -C /var/lib/grafana/plugins/
-                  rm /var/lib/grafana/plugins/plugin.tar.gz
+                  curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
+                  tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
+                  rm /var/lib/grafana/plugins/vl-plugin.tar.gz
               volumeMounts:
                 - name: grafana-data
                   mountPath: /var/lib/grafana
@@ -215,9 +215,9 @@ This example uses init container to download and install plugin.
 
    ``` bash
    ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
-   curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/plugin.tar.gz
-   tar -xf /var/lib/grafana/plugins/plugin.tar.gz -C /var/lib/grafana/plugins/
-   rm /var/lib/grafana/plugins/plugin.tar.gz
+   curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
+   tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
+   rm /var/lib/grafana/plugins/vl-plugin.tar.gz
    ```
 
 1. Restart Grafana
diff --git a/docs/anomaly-detection/components/models.md b/docs/anomaly-detection/components/models.md
index 1549fb4ad..eddec6d58 100644
--- a/docs/anomaly-detection/components/models.md
+++ b/docs/anomaly-detection/components/models.md
@@ -617,7 +617,7 @@ Here in this guide, we will
 > **Note**: By default, each custom model is created as [**univariate**](#univariate-models) / [**non-rolling**](#non-rolling-models) model. If you want to override this behavior, define models inherited from `RollingModel` (to get a rolling model), or having `is_multivariate` class arg set to `True` (please refer to the code example below).
 
 We'll create `custom_model.py` file with `CustomModel` class that will inherit from `vmanomaly`'s `Model` base class.
-In the `CustomModel` class there should be three required methods - `__init__`, `fit` and `infer`:
+In the `CustomModel` class, the following methods are required: - `__init__`, `fit`, `infer`, `serialize` and `deserialize`:
 * `__init__` method should initiate parameters for the model.
 
   **Note**: if your model relies on configs that have `arg` [key-value pair argument](./models.md#section-overview), do not forget to use Python's `**kwargs` in method's signature and to explicitly call
@@ -628,6 +628,8 @@ In the `CustomModel` class there should be three required methods - `__init__`,
   to initialize the base class each model derives from
 * `fit` method should contain the model training process. Please be aware that for `RollingModel` defining `fit` method is not needed, as the whole fit/infer process should be defined completely in `infer` method.
 * `infer` should return Pandas.DataFrame object with model's inferences.
+* `serialize` method that saves the model on disk.
+* `deserialize` load the saved model from disk.
 
 For the sake of simplicity, the model in this example will return one of two values of `anomaly_score` - 0 or 1 depending on input parameter `percentage`.
 
@@ -637,45 +639,56 @@ import numpy as np
 import pandas as pd
 import scipy.stats as st
 import logging
+from pickle import dumps
 
-from model.model import Model
+from model.model import (
+  PICKLE_PROTOCOL,
+  Model,
+  deserialize_basic
+)
 # from model.model import RollingModel  # inherit from it for your model to be of rolling type
 logger = logging.getLogger(__name__)
 
 
 class CustomModel(Model):
-    """
-    Custom model implementation.
-    """
-
-    # by default, each `Model` will be created as a univariate one
-    # uncomment line below for it to be of multivariate type
-    # is_multivariate = True
-
-    def __init__(self, percentage: float = 0.95, **kwargs):
-        super().__init__(**kwargs)
-        self.percentage = percentage
-        self._mean = np.nan
-        self._std = np.nan
-
-    def fit(self, df: pd.DataFrame):
-        # Model fit process: 
-        y = df['y']
-        self._mean = np.mean(y)
-        self._std = np.std(y)
-        if self._std == 0.0:
-            self._std = 1 / 65536
-
-    def infer(self, df: pd.DataFrame) -> np.array:
-        # Inference process:
-        y = df['y']
-        zscores = (y - self._mean) / self._std
-        anomaly_score_cdf = st.norm.cdf(np.abs(zscores))
-        df_pred = df[['timestamp', 'y']].copy()
-        df_pred['anomaly_score'] = anomaly_score_cdf > self.percentage
-        df_pred['anomaly_score'] = df_pred['anomaly_score'].astype('int32', errors='ignore')
-
-        return df_pred
+  """
+  Custom model implementation.
+  """
+  # by default, each `Model` will be created as a univariate one
+  # uncomment line below for it to be of multivariate type
+  #`is_multivariate = True`
+  
+  def __init__(self, percentage: float = 0.95, **kwargs):
+    super().__init__(**kwargs)
+    self.percentage = percentage
+    self._mean = np.nan
+    self._std = np.nan
+
+  def fit(self, df: pd.DataFrame):
+    # Model fit process:
+    y = df['y']
+    self._mean = np.mean(y)
+    self._std = np.std(y)
+    if self._std == 0.0:
+      self._std = 1 / 65536
+
+  def infer(self, df: pd.DataFrame) -> np.array:
+    # Inference process:
+    y = df['y']
+    zscores = (y - self._mean) / self._std
+    anomaly_score_cdf = st.norm.cdf(np.abs(zscores))
+    df_pred = df[['timestamp', 'y']].copy()
+    df_pred['anomaly_score'] = anomaly_score_cdf > self.percentage
+    df_pred['anomaly_score'] = df_pred['anomaly_score'].astype('int32', errors='ignore')
+
+    return df_pred
+
+    def serialize(self) -> None:
+      return dumps(self, protocol=PICKLE_PROTOCOL)
+
+    @staticmethod
+    def deserialize(model: str | bytes) -> 'CustomModel':
+      return deserialize_basic(model)
 ```
 
 
@@ -694,19 +707,19 @@ schedulers:
 
 models:
   custom_model:
-    # note: every custom model should implement this exact path, specified in `class` field
     class: "custom"  # or 'model.model.CustomModel' until v1.13.0
-    # custom model params are defined here
     percentage: 0.9
 
+
 reader:
-  datasource_url: "http://localhost:8428/"
+  datasource_url: "http://victoriametrics:8428/"
+  sampling_period: '1m'
   queries:
     ingestion_rate: 'sum(rate(vm_rows_inserted_total)) by (type)'
     churn_rate: 'sum(rate(vm_new_timeseries_created_total[5m]))'
 
 writer:
-  datasource_url: "http://localhost:8428/"
+  datasource_url: "http://victoriametrics:8428/"
   metric_format:
     __name__: "custom_$VAR"
     for: "$QUERY_KEY"
@@ -717,7 +730,7 @@ monitoring:
   pull:
     port: 8080
   push:
-    url: "http://localhost:8428/"
+    url: "http://victoriametrics:8428/"
     extra_labels:
       job: "vmanomaly-develop"
       config: "custom.yaml"
@@ -735,14 +748,15 @@ Now we can run the docker container putting as volumes both config and model fil
 
 > **Note**: place the model file to `/model/custom.py` path when copying
 
+./custom_model.py:/vmanomaly/model/custom.py
+
 ```sh
 docker run -it \
---net [YOUR_NETWORK] \
--v [YOUR_LICENSE_FILE_PATH]:/license.txt \
--v $(PWD)/custom_model.py:/vmanomaly/src/model/custom.py \
+-v $(PWD)/license:/license \
+-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
 victoriametrics/vmanomaly:latest /config.yaml \
---license-file=/license.txt
+--license-file=/license
 ```
 
 Please find more detailed instructions (license, etc.) [here](/anomaly-detection/overview.html#run-vmanomaly-docker-container)
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index 6f50ffbd2..9a959d833 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -1547,28 +1547,28 @@ scrape_configs:
 
 ```
 
-Each discovered target has an [`__address__`](https://docs.victoriametrics.com/relabeling.html#how-to-modify-scrape-urls-in-targets) label set
+Each discovered target has an [`__address__`](https://docs.victoriametrics.com/relabeling/#how-to-modify-scrape-urls-in-targets) label set
 to `<FQDN>:<port>`, where FQDN is discovered instance address and `<port>` is the port from the `vultr_sd_configs` (default port is `80`).
 
-The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling):
-
-* `__meta_vultr_instance_id`: A unique ID for the VPS Instance.
-* `__meta_vultr_instance_label`: The user-supplied label for this instance.
-* `__meta_vultr_instance_os`: The [Operating System name](https://www.vultr.com/api/#operation/list-os).
-* `__meta_vultr_instance_os_id`: The [Operating System id](https://www.vultr.com/api/#operation/list-os) used by this instance.
-* `__meta_vultr_instance_region`: The [Region id](https://www.vultr.com/api/#operation/list-regions) where the Instance is located.
-* `__meta_vultr_instance_plan`: A unique ID for the Plan.
-* `__meta_vultr_instance_main_ip`: The main IPv4 address.
-* `__meta_vultr_instance_internal_ip`: The internal IP used by this instance, if set. Only relevant when a VPC is attached.
-* `__meta_vultr_instance_main_ipv6`: The main IPv6 network address.
-* `__meta_vultr_instance_hostname`: The hostname for this instance.
-* `__meta_vultr_instance_server_status`: The server health status, which could be `none`, `locked`, `installingbooting`, `ok`.
-* `__meta_vultr_instance_vcpu_count`: Number of vCPUs.
-* `__meta_vultr_instance_ram_mb`: The amount of RAM in MB.
-* `__meta_vultr_instance_allowed_bandwidth_gb`: Monthly bandwidth quota in GB.
-* `__meta_vultr_instance_disk_gb`: The size of the disk in GB.
-* `__meta_vultr_instance_features`: "auto_backups", "ipv6", "ddos_protection".
-* `__meta_vultr_instance_tags`: Tags to apply to the instance.
+The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent/#relabeling):
+
+* `__meta_vultr_instance_allowed_bandwidth_gb`: monthly bandwidth quota in GB.
+* `__meta_vultr_instance_disk_gb`: the size of the disk in GB.
+* `__meta_vultr_instance_features`: comma-separated list of features avabilable to instance, such as "auto_backups", "ipv6", "ddos_protection".
+* `__meta_vultr_instance_hostname`: hostname for this instance.
+* `__meta_vultr_instance_id`: unique ID for the VPS Instance.
+* `__meta_vultr_instance_internal_ip`: internal IP used by this instance, if set. Only relevant when a VPC is attached.
+* `__meta_vultr_instance_label`: user-supplied label for this instance.
+* `__meta_vultr_instance_main_ip`: main IPv4 address.
+* `__meta_vultr_instance_main_ipv6`: main IPv6 network address.
+* `__meta_vultr_instance_os`: [operating System name](https://www.vultr.com/api/#operation/list-os).
+* `__meta_vultr_instance_os_id`: [operating System id](https://www.vultr.com/api/#operation/list-os) used by this instance.
+* `__meta_vultr_instance_plan`: unique ID for the Plan.
+* `__meta_vultr_instance_ram_mb`: the amount of RAM in MB.
+* `__meta_vultr_instance_region`: [region id](https://www.vultr.com/api/#operation/list-regions) where the Instance is located.
+* `__meta_vultr_instance_server_status`: server health status, which could be `none`, `locked`, `installingbooting`, `ok`.
+* `__meta_vultr_instance_tags`: comma-separated list of tags applied to the instance.
+* `__meta_vultr_instance_vcpu_count`: the number of vCPUs.
 
 The list of discovered Vultr targets is refreshed at the interval, which can be configured via `-promscrape.vultrSDCheckInterval` command-line flag, default: 30s.
 
diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md
index 68ee15830..e1acefe50 100644
--- a/docs/stream-aggregation.md
+++ b/docs/stream-aggregation.md
@@ -137,7 +137,7 @@ In this case it may be a good idea to drop the aggregated data during the first
 just after the restart of `vmagent` or single-node VictoriaMetrics. This can be done via the following options:
 
 - The `-streamAggr.ignoreFirstIntervals=N` command-line flag at `vmagent` and single-node VictoriaMetrics. This flag instructs skipping the first `N`
-  [aggregation intervals](#stream-aggregation-config) just after the restart accross all the [configured stream aggregation configs](#configuration).
+  [aggregation intervals](#stream-aggregation-config) just after the restart across all the [configured stream aggregation configs](#configuration).
 
   The `-remoteWrite.streamAggr.ignoreFirstIntervals=N` command-line flag can be specified individually per each `-remoteWrite.url` at [vmagent](https://docs.victoriametrics.com/vmagent/).
 
@@ -710,7 +710,7 @@ See also [quantiles](#quantiles), [min](#min), [max](#max) and [avg](#avg).
 
 `last` returns the last input [sample value](https://docs.victoriametrics.com/keyconcepts/#raw-samples) over the given `interval`.
 
-The results of `last` is roughly equal to the the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
+The results of `last` is roughly equal to the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
 
 ```metricsql
 last_over_time(some_metric[interval])
@@ -804,7 +804,7 @@ See also [count_samples](#count_samples) and [count_series](#count_series).
 `total` generates output [counter](https://docs.victoriametrics.com/keyconcepts/#counter) by summing the input counters over the given `interval`.
 `total` makes sense only for aggregating [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
 
-The results of `total` is roughly equal to the the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
+The results of `total` is roughly equal to the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
 
 ```metricsql
 sum(running_sum(increase_pure(some_counter)))
@@ -841,7 +841,7 @@ See also [total_prometheus](#total_prometheus), [increase](#increase) and [incre
 `total_prometheus` generates output [counter](https://docs.victoriametrics.com/keyconcepts/#counter) by summing the input counters over the given `interval`.
 `total_prometheus` makes sense only for aggregating [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
 
-The results of `total_prometheus` is roughly equal to the the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
+The results of `total_prometheus` is roughly equal to the following [MetricsQL](https://docs.victoriametrics.com/metricsql/) query:
 
 ```metricsql
 sum(running_sum(increase_prometheus(some_counter)))
diff --git a/docs/victoriametrics-datasource.md b/docs/victoriametrics-datasource.md
index 8de26e4b6..dde434b6e 100644
--- a/docs/victoriametrics-datasource.md
+++ b/docs/victoriametrics-datasource.md
@@ -159,7 +159,7 @@ Option 2. Using Grafana plugins section in `values.yaml`:
 
 ``` yaml
 plugins:
-  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.8.2/victoriametrics-datasource-v0.8.2zip;victoriametrics-datasource
+  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.8.2/victoriametrics-datasource-v0.8.2.zip;victoriametrics-datasource
 ```
 
 Option 3. Using init container:
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 8144f1e8e..69b5d2c6d 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -313,7 +313,7 @@ type ScrapeConfig struct {
 	NomadSDConfigs        []nomad.SDConfig        `yaml:"nomad_sd_configs,omitempty"`
 	OpenStackSDConfigs    []openstack.SDConfig    `yaml:"openstack_sd_configs,omitempty"`
 	StaticConfigs         []StaticConfig          `yaml:"static_configs,omitempty"`
-	VultrConfigs          []vultr.SDConfig        `yaml:"vultr_configs,omitempty"`
+	VultrSDConfigs        []vultr.SDConfig        `yaml:"vultr_configs,omitempty"`
 	YandexCloudSDConfigs  []yandexcloud.SDConfig  `yaml:"yandexcloud_sd_configs,omitempty"`
 
 	// These options are supported only by lib/promscrape.
@@ -394,8 +394,11 @@ func (sc *ScrapeConfig) mustStop() {
 	for i := range sc.OpenStackSDConfigs {
 		sc.OpenStackSDConfigs[i].MustStop()
 	}
-	for i := range sc.VultrConfigs {
-		sc.VultrConfigs[i].MustStop()
+	for i := range sc.VultrSDConfigs {
+		sc.VultrSDConfigs[i].MustStop()
+	}
+	for i := range sc.YandexCloudSDConfigs {
+		sc.YandexCloudSDConfigs[i].MustStop()
 	}
 }
 
@@ -757,8 +760,8 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 // getVultrSDScrapeWork returns `vultr_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getVultrSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
-		for i := range sc.VultrConfigs {
-			visitor(&sc.VultrConfigs[i])
+		for i := range sc.VultrSDConfigs {
+			visitor(&sc.VultrSDConfigs[i])
 		}
 	}
 	return cfg.getScrapeWorkGeneric(visitConfigs, "vultr_sd_config", prev)
diff --git a/lib/promscrape/discovery/vultr/api.go b/lib/promscrape/discovery/vultr/api.go
index 86e82df3a..fd32e54ce 100644
--- a/lib/promscrape/discovery/vultr/api.go
+++ b/lib/promscrape/discovery/vultr/api.go
@@ -2,6 +2,7 @@ package vultr
 
 import (
 	"fmt"
+	"net/url"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
@@ -11,22 +12,7 @@ type apiConfig struct {
 	c    *discoveryutils.Client
 	port int
 
-	listParams
-}
-
-// listParams is the query params of vultr ListInstance API.
-type listParams struct {
-	// paging params are not exposed to user, they will be filled
-	// dynamically during request. See `getInstances`.
-	// perPage int
-	// cursor string
-
-	// API query params for filtering.
-	label           string
-	mainIP          string
-	region          string
-	firewallGroupID string
-	hostname        string
+	listQueryParams string
 }
 
 // getAPIConfig get or create API config from configMap.
@@ -48,6 +34,10 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	// See: https://www.vultr.com/api/
 	apiServer := "https://api.vultr.com"
 
+	if sdc.HTTPClientConfig.BearerToken == nil {
+		return nil, fmt.Errorf("missing `bearer_token` option")
+	}
+
 	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
@@ -61,16 +51,31 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot create client for %q: %w", apiServer, err)
 	}
+
+	// Prepare additional query params for list instance API.
+	// See https://www.vultr.com/api/#tag/instances/operation/list-instances
+	var qp url.Values
+	if sdc.Label != "" {
+		qp.Set("label", sdc.Label)
+	}
+	if sdc.MainIP != "" {
+		qp.Set("main_ip", sdc.MainIP)
+	}
+	if sdc.Region != "" {
+		qp.Set("region", sdc.Region)
+	}
+	if sdc.FirewallGroupID != "" {
+		qp.Set("firewall_group_id", sdc.FirewallGroupID)
+	}
+	if sdc.Hostname != "" {
+		qp.Set("hostname", sdc.Hostname)
+	}
+
 	cfg := &apiConfig{
 		c:    c,
 		port: port,
-		listParams: listParams{
-			label:           sdc.Label,
-			mainIP:          sdc.MainIP,
-			region:          sdc.Region,
-			firewallGroupID: sdc.FirewallGroupID,
-			hostname:        sdc.Hostname,
-		},
+
+		listQueryParams: qp.Encode(),
 	}
 	return cfg, nil
 }
diff --git a/lib/promscrape/discovery/vultr/instance.go b/lib/promscrape/discovery/vultr/instance.go
index e510b7353..a1f546b4f 100644
--- a/lib/promscrape/discovery/vultr/instance.go
+++ b/lib/promscrape/discovery/vultr/instance.go
@@ -4,21 +4,20 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 // ListInstanceResponse is the response structure of Vultr ListInstance API.
 type ListInstanceResponse struct {
 	Instances []Instance `json:"instances"`
-	Meta      *Meta      `json:"Meta"`
+	Meta      Meta       `json:"meta"`
 }
 
 // Instance represents Vultr Instance (VPS).
+//
 // See: https://github.com/vultr/govultr/blob/5125e02e715ae6eb3ce854f0e7116c7ce545a710/instance.go#L81
 type Instance struct {
 	ID               string   `json:"id"`
-	Os               string   `json:"os"`
+	OS               string   `json:"os"`
 	RAM              int      `json:"ram"`
 	Disk             int      `json:"disk"`
 	MainIP           string   `json:"main_ip"`
@@ -30,39 +29,22 @@ type Instance struct {
 	Hostname         string   `json:"hostname"`
 	Label            string   `json:"label"`
 	InternalIP       string   `json:"internal_ip"`
-	OsID             int      `json:"os_id"`
+	OSID             int      `json:"os_id"`
 	Features         []string `json:"features"`
 	Plan             string   `json:"plan"`
 	Tags             []string `json:"tags"`
-
-	// The following fields are defined in the response but are not used during service discovery.
-	//DefaultPassword string `json:"default_password,omitempty"`
-	//DateCreated     string `json:"date_created"`
-	//Status          string `json:"status"`
-	//PowerStatus     string `json:"power_status"`
-	//NetmaskV4       string `json:"netmask_v4"`
-	//GatewayV4       string `json:"gateway_v4"`
-	//V6Network       string `json:"v6_network"`
-	//V6NetworkSize   int    `json:"v6_network_size"`
-	//// Deprecated: Tag should no longer be used. Instead, use Tags.
-	//Tag             string `json:"tag"`
-	//KVM             string `json:"kvm"`
-	//AppID           int    `json:"app_id"`
-	//ImageID         string `json:"image_id"`
-	//FirewallGroupID string `json:"firewall_group_id"`
-	//UserScheme      string `json:"user_scheme"`
 }
 
 // Meta represents the available pagination information
+//
+// See https://www.vultr.com/api/#section/Introduction/Meta-and-Pagination
 type Meta struct {
-	Total int `json:"total"`
-	Links *Links
+	Links Links `json:"links"`
 }
 
 // Links represent the next/previous cursor in your pagination calls
 type Links struct {
 	Next string `json:"next"`
-	Prev string `json:"prev"`
 }
 
 // getInstances retrieve instance from Vultr HTTP API.
@@ -70,39 +52,30 @@ func getInstances(cfg *apiConfig) ([]Instance, error) {
 	var instances []Instance
 
 	// prepare GET params
-	params := url.Values{}
-	params.Set("per_page", "100")
-	params.Set("label", cfg.label)
-	params.Set("main_ip", cfg.mainIP)
-	params.Set("region", cfg.region)
-	params.Set("firewall_group_id", cfg.firewallGroupID)
-	params.Set("hostname", cfg.hostname)
+	queryParams := cfg.listQueryParams
 
 	// send request to vultr API
 	for {
 		// See: https://www.vultr.com/api/#tag/instances/operation/list-instances
-		path := fmt.Sprintf("/v2/instances?%s", params.Encode())
-		resp, err := cfg.c.GetAPIResponse(path)
+		path := "/v2/instances?" + queryParams + "&per_page=100"
+		data, err := cfg.c.GetAPIResponse(path)
 		if err != nil {
-			logger.Errorf("get response from vultr failed, path:%s, err: %v", path, err)
-			return nil, err
+			return nil, fmt.Errorf("cannot get Vultr response from %q: %w", path, err)
 		}
 
-		var listInstanceResp ListInstanceResponse
-		if err = json.Unmarshal(resp, &listInstanceResp); err != nil {
-			logger.Errorf("unmarshal response from vultr failed, err: %v", err)
-			return nil, err
+		var resp ListInstanceResponse
+		if err := json.Unmarshal(data, &resp); err != nil {
+			return nil, fmt.Errorf("cannot unmarshal ListInstanceResponse obtained from %q: %w; response=%q", path, err, data)
 		}
 
-		instances = append(instances, listInstanceResp.Instances...)
+		instances = append(instances, resp.Instances...)
 
-		if listInstanceResp.Meta != nil && listInstanceResp.Meta.Links != nil && listInstanceResp.Meta.Links.Next != "" {
-			// if `next page` is available, set the cursor param and request again.
-			params.Set("cursor", listInstanceResp.Meta.Links.Next)
-		} else {
-			// otherwise exit the loop
+		if resp.Meta.Links.Next == "" {
 			break
 		}
+
+		// if `next page` is available, set the cursor param and request again.
+		queryParams = cfg.listQueryParams + "&cursor=" + url.QueryEscape(resp.Meta.Links.Next)
 	}
 
 	return instances, nil
diff --git a/lib/promscrape/discovery/vultr/vultr.go b/lib/promscrape/discovery/vultr/vultr.go
index d8999b2bb..13de311db 100644
--- a/lib/promscrape/discovery/vultr/vultr.go
+++ b/lib/promscrape/discovery/vultr/vultr.go
@@ -13,16 +13,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
 )
 
-const (
-	separator = ","
-)
-
-// SDCheckInterval defines interval for docker targets refresh.
+// SDCheckInterval defines interval for Vultr targets refresh.
 var SDCheckInterval = flag.Duration("promscrape.vultrSDCheckInterval", 30*time.Second, "Interval for checking for changes in Vultr. "+
 	"This works only if vultr_sd_configs is configured in '-promscrape.config' file. "+
 	"See https://docs.victoriametrics.com/sd_configs.html#vultr_sd_configs for details")
 
 // SDConfig represents service discovery config for Vultr.
+//
 // See: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#vultr_sd_config
 // Additional query params are supported, while Prometheus only supports `Port` and HTTP auth.
 type SDConfig struct {
@@ -62,7 +59,7 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 
 // MustStop stops further usage for sdc.
 func (sdc *SDConfig) MustStop() {
-	configMap.Delete(sdc)
+	_ = configMap.Delete(sdc)
 }
 
 // getInstanceLabels returns labels for vultr instances obtained from the given cfg
@@ -72,34 +69,36 @@ func getInstanceLabels(instances []Instance, port int) []*promutils.Labels {
 	for _, instance := range instances {
 		m := promutils.NewLabels(18)
 		m.Add("__address__", discoveryutils.JoinHostPort(instance.MainIP, port))
+		m.Add("__meta_vultr_instance_allowed_bandwidth_gb", strconv.Itoa(instance.AllowedBandwidth))
+		m.Add("__meta_vultr_instance_disk_gb", strconv.Itoa(instance.Disk))
+		m.Add("__meta_vultr_instance_hostname", instance.Hostname)
 		m.Add("__meta_vultr_instance_id", instance.ID)
+		m.Add("__meta_vultr_instance_internal_ip", instance.InternalIP)
 		m.Add("__meta_vultr_instance_label", instance.Label)
-		m.Add("__meta_vultr_instance_os", instance.Os)
-		m.Add("__meta_vultr_instance_os_id", strconv.Itoa(instance.OsID))
-		m.Add("__meta_vultr_instance_region", instance.Region)
-		m.Add("__meta_vultr_instance_plan", instance.Plan)
 		m.Add("__meta_vultr_instance_main_ip", instance.MainIP)
-		m.Add("__meta_vultr_instance_internal_ip", instance.InternalIP)
 		m.Add("__meta_vultr_instance_main_ipv6", instance.V6MainIP)
-		m.Add("__meta_vultr_instance_hostname", instance.Hostname)
+		m.Add("__meta_vultr_instance_os", instance.OS)
+		m.Add("__meta_vultr_instance_os_id", strconv.Itoa(instance.OSID))
+		m.Add("__meta_vultr_instance_plan", instance.Plan)
+		m.Add("__meta_vultr_instance_region", instance.Region)
+		m.Add("__meta_vultr_instance_ram_mb", strconv.Itoa(instance.RAM))
 		m.Add("__meta_vultr_instance_server_status", instance.ServerStatus)
 		m.Add("__meta_vultr_instance_vcpu_count", strconv.Itoa(instance.VCPUCount))
-		m.Add("__meta_vultr_instance_ram_mb", strconv.Itoa(instance.RAM))
-		m.Add("__meta_vultr_instance_allowed_bandwidth_gb", strconv.Itoa(instance.AllowedBandwidth))
-		m.Add("__meta_vultr_instance_disk_gb", strconv.Itoa(instance.Disk))
 
-		// We surround the separated list with the separator as well. This way regular expressions
-		// in relabeling rules don't have to consider feature positions.
 		if len(instance.Features) > 0 {
-			features := separator + strings.Join(instance.Features, separator) + separator
-			m.Add("__meta_vultr_instance_features", features)
+			m.Add("__meta_vultr_instance_features", joinStrings(instance.Features))
 		}
 
 		if len(instance.Tags) > 0 {
-			tags := separator + strings.Join(instance.Tags, separator) + separator
-			m.Add("__meta_vultr_instance_tags", tags)
+			m.Add("__meta_vultr_instance_tags", joinStrings(instance.Tags))
 		}
+
 		ms = append(ms, m)
 	}
+
 	return ms
 }
+
+func joinStrings(a []string) string {
+	return "," + strings.Join(a, ",") + ","
+}
diff --git a/lib/promscrape/discovery/yandexcloud/yandexcloud.go b/lib/promscrape/discovery/yandexcloud/yandexcloud.go
index 621e3c729..b3bf0c89d 100644
--- a/lib/promscrape/discovery/yandexcloud/yandexcloud.go
+++ b/lib/promscrape/discovery/yandexcloud/yandexcloud.go
@@ -38,6 +38,11 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 	}
 }
 
+// MustStop stops further usage for sdc.
+func (sdc *SDConfig) MustStop() {
+	_ = configMap.Delete(sdc)
+}
+
 func (cfg *apiConfig) getInstances(folderID string) ([]instance, error) {
 	instancesURL := cfg.serviceEndpoints["compute"] + "/compute/v1/instances"
 	instancesURL += "?folderId=" + url.QueryEscape(folderID)


Test patch:
diff --git a/lib/promscrape/discovery/vultr/api_test.go b/lib/promscrape/discovery/vultr/api_test.go
index 06e491446..7d33f4cf7 100644
--- a/lib/promscrape/discovery/vultr/api_test.go
+++ b/lib/promscrape/discovery/vultr/api_test.go
@@ -2,15 +2,30 @@ package vultr
 
 import (
 	"testing"
-)
 
-func TestNewAPIConfig(t *testing.T) {
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
 
+func TestNewAPIConfig_Failure(t *testing.T) {
 	sdc := &SDConfig{}
 	baseDir := "."
 	_, err := newAPIConfig(sdc, baseDir)
+	if err == nil {
+		t.Fatalf("expecting non-nil error")
+	}
+}
+
+func TestNewAPIConfig_Success(t *testing.T) {
+	sdc := &SDConfig{
+		HTTPClientConfig: promauth.HTTPClientConfig{
+			BearerToken: &promauth.Secret{
+				S: "foobar",
+			},
+		},
+	}
+	baseDir := "."
+	_, err := newAPIConfig(sdc, baseDir)
 	if err != nil {
-		t.Errorf("newAPIConfig failed with, err: %v", err)
-		return
+		t.Fatalf("newAPIConfig failed with, err: %v", err)
 	}
 }
diff --git a/lib/promscrape/discovery/vultr/instance_test.go b/lib/promscrape/discovery/vultr/instance_test.go
index b888c1a13..0a61db057 100644
--- a/lib/promscrape/discovery/vultr/instance_test.go
+++ b/lib/promscrape/discovery/vultr/instance_test.go
@@ -1,7 +1,6 @@
 package vultr
 
 import (
-	"errors"
 	"reflect"
 	"testing"
 
@@ -9,147 +8,65 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
-// TestGetInstances runs general test cases for GetInstances
-func TestGetInstances(t *testing.T) {
-	testCases := []struct {
-		name           string
-		apiResponse    string
-		apiError       bool
-		expectError    bool
-		expectResponse []Instance
-	}{
-		{
-			name:           "success response",
-			apiResponse:    mockListInstanceSuccessResp,
-			apiError:       false,
-			expectError:    false,
-			expectResponse: expectSuccessInstances,
-		},
-		{
-			name:           "failed response",
-			apiResponse:    mockListInstanceFailedResp,
-			apiError:       true,
-			expectError:    true,
-			expectResponse: nil,
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			// Prepare a mock Vultr server.
-			mockServer := newMockVultrServer(func() ([]byte, error) {
-				var e error
-				if tt.apiError {
-					e = errors.New("mock error")
-				}
-				return []byte(tt.apiResponse), e
-			})
-
-			// Prepare a discovery HTTP client who calls mock server.
-			client, _ := discoveryutils.NewClient(mockServer.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
-			cfg := &apiConfig{
-				c: client,
-			}
-
-			// execute `getInstances`
-			instances, err := getInstances(cfg)
-
-			// evaluate test result
-			if tt.expectError != (err != nil) {
-				t.Errorf("getInstances expect (error != nil): %t, got error: %v", tt.expectError, err)
-			}
-
-			if !reflect.DeepEqual(tt.expectResponse, instances) {
-				t.Errorf("getInstances expect result: %v, got: %v", tt.expectResponse, instances)
-			}
-		})
-	}
-}
-
-// TestGetInstancesPaging run test cases for response with multiple pages.
-func TestGetInstancesPaging(t *testing.T) {
-	// Prepare a mock Vultr server.
-	// requestCount control the mock response for different page request.
-	requestCount := 0
+func TestGetInstances_Success(t *testing.T) {
+	s := newMockVultrServer(func() []byte {
+		const resp = `{
+			"instances": [{
+			"id": "fake-id-07f7-4b68-88ac-fake-id",
+			"os": "Ubuntu 22.04 x64",
+			"ram": 1024,
+			"disk": 25,
+			"main_ip": "64.176.84.27",
+			"vcpu_count": 1,
+			"region": "sgp",
+			"plan": "vc2-1c-1gb",
+			"date_created": "2024-04-05T05:41:28+00:00",
+			"status": "active",
+			"allowed_bandwidth": 1,
+			"netmask_v4": "255.255.254.0",
+			"gateway_v4": "64.176.63.2",
+			"power_status": "running",
+			"server_status": "installingbooting",
+			"v6_network": "2002:18f0:4100:263a::",
+			"v6_main_ip": "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			"v6_network_size": 64,
+			"label": "vultr-sd",
+			"internal_ip": "",
+			"kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=secret_data_string",
+			"hostname": "vultr-sd",
+			"tag": "",
+			"tags": [],
+			"os_id": 1743,
+			"app_id": 0,
+			"image_id": "",
+			"firewall_group_id": "",
+			"features": ["ipv6"],
+			"user_scheme": "root"
+		}]
+	}`
 
-	mockServer := newMockVultrServer(func() ([]byte, error) {
-		// for the 1st request, response with `next` cursor
-		if requestCount == 0 {
-			requestCount++
-			return []byte(mockListInstanceSuccessPage0Resp), nil
-		}
-		// for the 2nd+ request, response with `prev` cursor and empty `next`.
-		return []byte(mockListInstanceSuccessPage1Resp), nil
+		return []byte(resp)
 	})
 
 	// Prepare a discovery HTTP client who calls mock server.
-	client, _ := discoveryutils.NewClient(mockServer.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	client, err := discoveryutils.NewClient(s.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	if err != nil {
+		t.Fatalf("unexpected error wen creating http client: %s", err)
+	}
 	cfg := &apiConfig{
 		c: client,
 	}
 
 	// execute `getInstances`
 	instances, err := getInstances(cfg)
-
-	// evaluate test result
 	if err != nil {
-		t.Errorf("getInstances expect error: %v, got error: %v", nil, err)
+		t.Fatalf("unexpected error in getInstances(): %s", err)
 	}
 
-	if !reflect.DeepEqual(expectSuccessPagingInstances, instances) {
-		t.Errorf("getInstances expect result: %v, got: %v", expectSuccessPagingInstances, instances)
-	}
-}
-
-// ------------ Test dataset ------------
-var (
-	// mockListInstanceSuccessResp is crawled from a real-world response of ListInstance API
-	// with sensitive info removed/modified.
-	mockListInstanceSuccessResp = `{
-	"instances": [{
-		"id": "fake-id-07f7-4b68-88ac-fake-id",
-		"os": "Ubuntu 22.04 x64",
-		"ram": 1024,
-		"disk": 25,
-		"main_ip": "64.176.84.27",
-		"vcpu_count": 1,
-		"region": "sgp",
-		"plan": "vc2-1c-1gb",
-		"date_created": "2024-04-05T05:41:28+00:00",
-		"status": "active",
-		"allowed_bandwidth": 1,
-		"netmask_v4": "255.255.254.0",
-		"gateway_v4": "64.176.63.2",
-		"power_status": "running",
-		"server_status": "installingbooting",
-		"v6_network": "2002:18f0:4100:263a::",
-		"v6_main_ip": "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
-		"v6_network_size": 64,
-		"label": "vultr-sd",
-		"internal_ip": "",
-		"kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=secret_data_string",
-		"hostname": "vultr-sd",
-		"tag": "",
-		"tags": [],
-		"os_id": 1743,
-		"app_id": 0,
-		"image_id": "",
-		"firewall_group_id": "",
-		"features": ["ipv6"],
-		"user_scheme": "root"
-	}],
-	"meta": {
-		"total": 1,
-		"links": {
-			"next": "",
-			"prev": ""
-		}
-	}
-}`
-	expectSuccessInstances = []Instance{
+	expectedInstances := []Instance{
 		{
 			ID:               "fake-id-07f7-4b68-88ac-fake-id",
-			Os:               "Ubuntu 22.04 x64",
+			OS:               "Ubuntu 22.04 x64",
 			RAM:              1024,
 			Disk:             25,
 			MainIP:           "64.176.84.27",
@@ -163,21 +80,74 @@ var (
 			InternalIP:       "",
 			Hostname:         "vultr-sd",
 			Tags:             []string{},
-			OsID:             1743,
+			OSID:             1743,
 			Features:         []string{"ipv6"},
 		},
 	}
-)
+	if !reflect.DeepEqual(instances, expectedInstances) {
+		t.Fatalf("unexpected result\ngot\n%#v\nwant\n%#v", instances, expectedInstances)
+	}
+}
 
-var (
-	mockListInstanceFailedResp = `{"error":"Invalid API token.","status":401}`
-)
+func TestGetInstances_Failure(t *testing.T) {
+	s := newMockVultrServer(func() []byte {
+		return []byte("some error")
+	})
+
+	// Prepare a discovery HTTP client who calls mock server.
+	client, err := discoveryutils.NewClient(s.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	if err != nil {
+		t.Fatalf("unexpected error wen creating http client: %s", err)
+	}
+	cfg := &apiConfig{
+		c: client,
+	}
+
+	// execute `getInstances`
+	if _, err := getInstances(cfg); err == nil {
+		t.Fatalf("expecting non-nil error from getInstances()")
+	}
+}
 
+func TestGetInstances_Paging(t *testing.T) {
+	// Prepare a mock Vultr server.
+	// requestCount control the mock response for different page request.
+	requestCount := 0
+	s := newMockVultrServer(func() []byte {
+		// for the 1st request, response with `next` cursor
+		if requestCount == 0 {
+			requestCount++
+			return []byte(mockListInstanceSuccessPage0Resp)
+		}
+		// for the 2nd+ request, response with empty `next`.
+		return []byte(mockListInstanceSuccessPage1Resp)
+	})
+
+	// Prepare a discovery HTTP client who calls mock server.
+	client, err := discoveryutils.NewClient(s.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	if err != nil {
+		t.Fatalf("unexpected error wen creating http client: %s", err)
+	}
+	cfg := &apiConfig{
+		c: client,
+	}
+
+	// execute `getInstances`
+	instances, err := getInstances(cfg)
+	if err != nil {
+		t.Fatalf("unexpected error in getInstances(): %s", err)
+	}
+	if !reflect.DeepEqual(expectSuccessPagingInstances, instances) {
+		t.Fatalf("unexpected getInstances() result\ngot\n%#v\nwant\n%#v", instances, expectSuccessPagingInstances)
+	}
+}
+
+// ------------ Test dataset ------------
 var (
 	// mockListInstanceSuccessPage0Resp contains `next` cursor
 	mockListInstanceSuccessPage0Resp = `{
 	"instances": [{
-		"id": "fake-id-07f7-4b68-88ac-fake-id",
+		"id": "1-fake-id-07f7-4b68-88ac-fake-id",
 		"os": "Ubuntu 22.04 x64",
 		"ram": 1024,
 		"disk": 25,
@@ -209,17 +179,15 @@ var (
 		"user_scheme": "root"
 	}],
 	"meta": {
-		"total": 2,
 		"links": {
-			"next": "fake-cursor-string",
-			"prev": ""
+			"next": "fake-cursor-string"
 		}
 	}
 }`
-	// mockListInstanceSuccessPage1Resp contains `prev` cursor
+	// mockListInstanceSuccessPage1Resp contains empty 'next' cursor
 	mockListInstanceSuccessPage1Resp = `{
 	"instances": [{
-		"id": "fake-id-07f7-4b68-88ac-fake-id",
+		"id": "2-fake-id-07f7-4b68-88ac-fake-id",
 		"os": "Ubuntu 22.04 x64",
 		"ram": 1024,
 		"disk": 25,
@@ -251,17 +219,15 @@ var (
 		"user_scheme": "root"
 	}],
 	"meta": {
-		"total": 2,
 		"links": {
-			"next": "",
-			"prev": "fake-cursor-string"
+			"next": ""
 		}
 	}
 }`
 	expectSuccessPagingInstances = []Instance{
 		{
-			ID:               "fake-id-07f7-4b68-88ac-fake-id",
-			Os:               "Ubuntu 22.04 x64",
+			ID:               "1-fake-id-07f7-4b68-88ac-fake-id",
+			OS:               "Ubuntu 22.04 x64",
 			RAM:              1024,
 			Disk:             25,
 			MainIP:           "64.176.84.27",
@@ -275,12 +241,12 @@ var (
 			InternalIP:       "",
 			Hostname:         "vultr-sd",
 			Tags:             []string{},
-			OsID:             1743,
+			OSID:             1743,
 			Features:         []string{"ipv6"},
 		},
 		{
-			ID:               "fake-id-07f7-4b68-88ac-fake-id",
-			Os:               "Ubuntu 22.04 x64",
+			ID:               "2-fake-id-07f7-4b68-88ac-fake-id",
+			OS:               "Ubuntu 22.04 x64",
 			RAM:              1024,
 			Disk:             25,
 			MainIP:           "64.176.84.27",
@@ -294,7 +260,7 @@ var (
 			InternalIP:       "",
 			Hostname:         "vultr-sd",
 			Tags:             []string{},
-			OsID:             1743,
+			OSID:             1743,
 			Features:         []string{"ipv6"},
 		},
 	}
diff --git a/lib/promscrape/discovery/vultr/mock_server_test.go b/lib/promscrape/discovery/vultr/mock_server_test.go
index c0dd79e46..1af362cc4 100644
--- a/lib/promscrape/discovery/vultr/mock_server_test.go
+++ b/lib/promscrape/discovery/vultr/mock_server_test.go
@@ -1,40 +1,23 @@
 package vultr
 
 import (
-	"fmt"
 	"net/http"
 	"net/http/httptest"
 )
 
-func newMockVultrServer(jsonResponse func() ([]byte, error)) *vultrServer {
-	rw := &vultrServer{}
-	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
-	rw.jsonResponse = jsonResponse
-	return rw
-}
-
-type vultrServer struct {
+type mockVultrServer struct {
 	*httptest.Server
-	jsonResponse func() ([]byte, error)
+	responseFunc func() []byte
 }
 
-func (rw *vultrServer) err(w http.ResponseWriter, err error) {
-	w.WriteHeader(http.StatusBadRequest)
-	w.Write([]byte(err.Error()))
+func newMockVultrServer(responseFunc func() []byte) *mockVultrServer {
+	var s mockVultrServer
+	s.responseFunc = responseFunc
+	s.Server = httptest.NewServer(http.HandlerFunc(s.handler))
+	return &s
 }
 
-func (rw *vultrServer) handler(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		rw.err(w, fmt.Errorf("bad method %q", r.Method))
-		return
-	}
-
-	resp, err := rw.jsonResponse()
-	if err != nil {
-		rw.err(w, err)
-		return
-	}
-
-	w.Write(resp)
-	w.WriteHeader(http.StatusOK)
+func (s *mockVultrServer) handler(w http.ResponseWriter, _ *http.Request) {
+	data := s.responseFunc()
+	w.Write(data)
 }
diff --git a/lib/promscrape/discovery/vultr/vultr_test.go b/lib/promscrape/discovery/vultr/vultr_test.go
index 572782726..849d326ba 100644
--- a/lib/promscrape/discovery/vultr/vultr_test.go
+++ b/lib/promscrape/discovery/vultr/vultr_test.go
@@ -11,7 +11,7 @@ func TestGetInstanceLabels(t *testing.T) {
 	input := []Instance{
 		{
 			ID:               "fake-id-07f7-4b68-88ac-fake-id",
-			Os:               "Ubuntu 22.04 x64",
+			OS:               "Ubuntu 22.04 x64",
 			RAM:              1024,
 			Disk:             25,
 			MainIP:           "64.176.84.27",
@@ -25,12 +25,12 @@ func TestGetInstanceLabels(t *testing.T) {
 			InternalIP:       "",
 			Hostname:         "vultr-sd",
 			Tags:             []string{"mock tags"},
-			OsID:             1743,
+			OSID:             1743,
 			Features:         []string{"ipv6"},
 		},
 		{
 			ID:               "fake-id-07f7-4b68-88ac-fake-id",
-			Os:               "Ubuntu 22.04 x64",
+			OS:               "Ubuntu 22.04 x64",
 			RAM:              1024,
 			Disk:             25,
 			MainIP:           "64.176.84.27",
@@ -44,7 +44,7 @@ func TestGetInstanceLabels(t *testing.T) {
 			InternalIP:       "",
 			Hostname:         "vultr-sd",
 			Tags:             []string{"mock tags"},
-			OsID:             1743,
+			OSID:             1743,
 			Features:         []string{"ipv6"},
 		},
 	}