Repository: VictoriaMetrics/VictoriaMetrics
Pull request #6238 (2024-05-08 01:20:33)

chore: [deployment] upgrade from go 1.22.2 to 1.22.3 to include security fixes
### Describe Your Changes

upgrade from go 1.22.2 to 1.22.3 to include security fixes. Also see:
- https://go.dev/doc/devel/release
- https://github.com/golang/go/issues?q=milestone%3AGo1.22.3+label%3ACherryPickApproved

### Checklist

The following checks are **mandatory**:

- [X] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).



Patch:
diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web
index f54841860..1ccad9a66 100644
--- a/app/vmui/Dockerfile-web
+++ b/app/vmui/Dockerfile-web
@@ -1,4 +1,4 @@
-FROM golang:1.22.2 as build-web-stage
+FROM golang:1.22.3 as build-web-stage
 COPY build /build
 
 WORKDIR /build
diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index 9ae67e9ae..929166cd3 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -5,7 +5,7 @@ DOCKER_NAMESPACE ?= victoriametrics
 ROOT_IMAGE ?= alpine:3.19.1
 CERTS_IMAGE := alpine:3.19.1
 
-GO_BUILDER_IMAGE := golang:1.22.2-alpine
+GO_BUILDER_IMAGE := golang:1.22.3-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 DOCKER ?= docker
diff --git a/deployment/logs-benchmark/docker-compose-elk.yml b/deployment/logs-benchmark/docker-compose-elk.yml
index 26fb5d9bf..40b02e50e 100644
--- a/deployment/logs-benchmark/docker-compose-elk.yml
+++ b/deployment/logs-benchmark/docker-compose-elk.yml
@@ -18,7 +18,7 @@ services:
       - vlogs
 
   generator:
-    image: golang:1.22.2-alpine
+    image: golang:1.22.3-alpine
     restart: always
     working_dir: /go/src/app
     volumes:
diff --git a/deployment/logs-benchmark/docker-compose-loki.yml b/deployment/logs-benchmark/docker-compose-loki.yml
index b090641f3..c20a8125c 100644
--- a/deployment/logs-benchmark/docker-compose-loki.yml
+++ b/deployment/logs-benchmark/docker-compose-loki.yml
@@ -2,7 +2,7 @@ version: '3'
 
 services:
   generator:
-    image: golang:1.22.2-alpine
+    image: golang:1.22.3-alpine
     restart: always
     working_dir: /go/src/app
     volumes:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 76fb4a3dd..9a83a07ed 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -30,11 +30,14 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 ## tip
 
+* SECURITY: upgrade Go builder from Go1.22.2 to Go1.22.3. See [the list of issues addressed in Go1.22.3](https://github.com/golang/go/issues?q=milestone%3AGo1.22.3+label%3ACherryPickApproved).
+
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): support selecting of multiple instances on the dashboard. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5869) for details.
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): properly display version in the Stats row for the custom builds of VictoriaMetrics.
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): add `Network Usage` panel to `Resource Usage` row.
 * FEATURE: [dashboards/operator](https://grafana.com/grafana/dashboards/17869), [dashboards/backupmanager](https://grafana.com/grafana/dashboards/17798) and [dashboard/tenant-statistic](https://grafana.com/grafana/dashboards/16399): update dashboard to be compatible with Grafana 10+ version.
 * FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add new panel `Concurrent selects` to `vmstorage` row. The panel will show how many ongoing select queries are processed by vmstorage and should help to identify resource bottlenecks. See panel description for more details.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add service discovery support for [Vultr](https://www.vultr.com/). See [these docs](https://docs.victoriametrics.com/sd_configs/#vultr_sd_configs) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6041).
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) release.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): prevent potential panic during [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) if more than one `--remoteWrite.streamAggr.dedupInterval` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6205).
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index f2357d020..eda9a4a8c 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -35,6 +35,7 @@ supports the following Prometheus-compatible service discovery options for Prome
 * `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs).
 * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs).
 * `static_configs` is for scraping statically defined targets. See [these docs](#static_configs).
+* `vultr_sd_configs` is for discovering and scraping [Vultr](https://www.vultr.com/) targets. See [these docs](#vultr_sd_configs).
 * `yandexcloud_sd_configs` is for discovering and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud_sd_configs).
 
 Note that the `refresh_interval` option isn't supported for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
@@ -1498,6 +1499,79 @@ scrape_configs:
 
 See [these examples](https://docs.victoriametrics.com/scrape_config_examples/#static-configs) on how to configure scraping for static targets.
 
+## vultr_sd_configs
+Vultr SD configuration discovers scrape targets from [Vultr](https://www.vultr.com/) Instances.
+
+Configuration example:
+
+```yaml
+scrape_configs:
+- job_name: vultr
+  vultr_sd_configs:
+
+    # bearer_token is a Bearer token to send in every HTTP API request during service discovery (mandatory).
+    # See: https://my.vultr.com/settings/#settingsapi
+  - bearer_token: "..."
+
+    # Vultr provides query arguments to filter instances.
+    # See: https://www.vultr.com/api/#tag/instances
+
+    # label is an optional query arguments to filter instances by label.
+    #
+    # label: "..."
+
+    # main_ip is an optional query arguments to filter instances by main ip address.
+    #
+    # main_ip: "..."
+
+    # region is an optional query arguments to filter instances by region id.
+    #
+    # region: "..."
+
+    # firewall_group_id is an optional query arguments to filter instances by firewall group id.
+    #
+    # firewall_group_id: "..."
+
+    # hostname is an optional query arguments to filter instances by hostname.
+    #
+    # hostname: "..."
+
+    # port is an optional port to scrape metrics from.
+    # By default, port 80 is used.
+    #
+    # port: ...
+
+    # Additional HTTP API client options can be specified here.
+    # See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options
+
+
+```
+
+Each discovered target has an [`__address__`](https://docs.victoriametrics.com/relabeling.html#how-to-modify-scrape-urls-in-targets) label set
+to `<FQDN>:<port>`, where FQDN is discovered instance address and `<port>` is the port from the `vultr_sd_configs` (default port is `80`).
+
+The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling):
+
+* `__meta_vultr_instance_id`: A unique ID for the VPS Instance.
+* `__meta_vultr_instance_label`: The user-supplied label for this instance.
+* `__meta_vultr_instance_os`: The [Operating System name](https://www.vultr.com/api/#operation/list-os).
+* `__meta_vultr_instance_os_id`: The [Operating System id](https://www.vultr.com/api/#operation/list-os) used by this instance.
+* `__meta_vultr_instance_region`: The [Region id](https://www.vultr.com/api/#operation/list-regions) where the Instance is located.
+* `__meta_vultr_instance_plan`: A unique ID for the Plan.
+* `__meta_vultr_instance_main_ip`: The main IPv4 address.
+* `__meta_vultr_instance_internal_ip`: The internal IP used by this instance, if set. Only relevant when a VPC is attached.
+* `__meta_vultr_instance_main_ipv6`: The main IPv6 network address.
+* `__meta_vultr_instance_hostname`: The hostname for this instance.
+* `__meta_vultr_instance_server_status`: The server health status, which could be `none`, `locked`, `installingbooting`, `ok`.
+* `__meta_vultr_instance_vcpu_count`: Number of vCPUs.
+* `__meta_vultr_instance_ram_mb`: The amount of RAM in MB.
+* `__meta_vultr_instance_allowed_bandwidth_gb`: Monthly bandwidth quota in GB.
+* `__meta_vultr_instance_disk_gb`: The size of the disk in GB.
+* `__meta_vultr_instance_features`: "auto_backups", "ipv6", "ddos_protection".
+* `__meta_vultr_instance_tags`: Tags to apply to the instance.
+
+The list of discovered Vultr targets is refreshed at the interval, which can be configured via `-promscrape.vultrSDCheckInterval` command-line flag, default: 30s.
+
 ## yandexcloud_sd_configs
 
 [Yandex Cloud](https://cloud.yandex.com/en/) SD configurations allow retrieving scrape targets from accessible folders.
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 4c7d8421b..7e368f437 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -1982,6 +1982,8 @@ See the docs at https://docs.victoriametrics.com/vmagent/ .
      Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
   -promscrape.suppressScrapeErrorsDelay duration
      The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
+  -promscrape.vultrSDCheckInterval duration
+     Interval for checking for changes in Vultr. This works only if vultr_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#vultr_sd_configs for details  (default 30s)
   -promscrape.yandexcloudSDCheckInterval duration
      Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#yandexcloud_sd_configs for details (default 30s)
   -pushmetrics.disableCompression
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 5516b60f1..b0ea9603f 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -36,6 +36,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kuma"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/vultr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
@@ -308,6 +309,7 @@ type ScrapeConfig struct {
 	NomadSDConfigs        []nomad.SDConfig        `yaml:"nomad_sd_configs,omitempty"`
 	OpenStackSDConfigs    []openstack.SDConfig    `yaml:"openstack_sd_configs,omitempty"`
 	StaticConfigs         []StaticConfig          `yaml:"static_configs,omitempty"`
+	VultrConfigs          []vultr.SDConfig        `yaml:"vultr_configs,omitempty"`
 	YandexCloudSDConfigs  []yandexcloud.SDConfig  `yaml:"yandexcloud_sd_configs,omitempty"`
 
 	// These options are supported only by lib/promscrape.
@@ -388,6 +390,9 @@ func (sc *ScrapeConfig) mustStop() {
 	for i := range sc.OpenStackSDConfigs {
 		sc.OpenStackSDConfigs[i].MustStop()
 	}
+	for i := range sc.VultrConfigs {
+		sc.VultrConfigs[i].MustStop()
+	}
 }
 
 // FileSDConfig represents file-based service discovery config.
@@ -745,6 +750,16 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	return cfg.getScrapeWorkGeneric(visitConfigs, "openstack_sd_config", prev)
 }
 
+// getVultrSDScrapeWork returns `vultr_sd_configs` ScrapeWork from cfg.
+func (cfg *Config) getVultrSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
+	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
+		for i := range sc.VultrConfigs {
+			visitor(&sc.VultrConfigs[i])
+		}
+	}
+	return cfg.getScrapeWorkGeneric(visitConfigs, "vultr_sd_config", prev)
+}
+
 // getYandexCloudSDScrapeWork returns `yandexcloud_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getYandexCloudSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
diff --git a/lib/promscrape/discovery/vultr/api.go b/lib/promscrape/discovery/vultr/api.go
new file mode 100644
index 000000000..86e82df3a
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/api.go
@@ -0,0 +1,76 @@
+package vultr
+
+import (
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+// apiConfig contains config for API server.
+type apiConfig struct {
+	c    *discoveryutils.Client
+	port int
+
+	listParams
+}
+
+// listParams is the query params of vultr ListInstance API.
+type listParams struct {
+	// paging params are not exposed to user, they will be filled
+	// dynamically during request. See `getInstances`.
+	// perPage int
+	// cursor string
+
+	// API query params for filtering.
+	label           string
+	mainIP          string
+	region          string
+	firewallGroupID string
+	hostname        string
+}
+
+// getAPIConfig get or create API config from configMap.
+func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
+	if err != nil {
+		return nil, err
+	}
+	return v.(*apiConfig), nil
+}
+
+// newAPIConfig create API Config.
+func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	port := sdc.Port
+	if port == 0 {
+		port = 80
+	}
+
+	// See: https://www.vultr.com/api/
+	apiServer := "https://api.vultr.com"
+
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
+	}
+	proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
+	}
+
+	c, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, &sdc.HTTPClientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create client for %q: %w", apiServer, err)
+	}
+	cfg := &apiConfig{
+		c:    c,
+		port: port,
+		listParams: listParams{
+			label:           sdc.Label,
+			mainIP:          sdc.MainIP,
+			region:          sdc.Region,
+			firewallGroupID: sdc.FirewallGroupID,
+			hostname:        sdc.Hostname,
+		},
+	}
+	return cfg, nil
+}
diff --git a/lib/promscrape/discovery/vultr/instance.go b/lib/promscrape/discovery/vultr/instance.go
new file mode 100644
index 000000000..e510b7353
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/instance.go
@@ -0,0 +1,109 @@
+package vultr
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// ListInstanceResponse is the response structure of Vultr ListInstance API.
+type ListInstanceResponse struct {
+	Instances []Instance `json:"instances"`
+	Meta      *Meta      `json:"Meta"`
+}
+
+// Instance represents Vultr Instance (VPS).
+// See: https://github.com/vultr/govultr/blob/5125e02e715ae6eb3ce854f0e7116c7ce545a710/instance.go#L81
+type Instance struct {
+	ID               string   `json:"id"`
+	Os               string   `json:"os"`
+	RAM              int      `json:"ram"`
+	Disk             int      `json:"disk"`
+	MainIP           string   `json:"main_ip"`
+	VCPUCount        int      `json:"vcpu_count"`
+	Region           string   `json:"region"`
+	ServerStatus     string   `json:"server_status"`
+	AllowedBandwidth int      `json:"allowed_bandwidth"`
+	V6MainIP         string   `json:"v6_main_ip"`
+	Hostname         string   `json:"hostname"`
+	Label            string   `json:"label"`
+	InternalIP       string   `json:"internal_ip"`
+	OsID             int      `json:"os_id"`
+	Features         []string `json:"features"`
+	Plan             string   `json:"plan"`
+	Tags             []string `json:"tags"`
+
+	// The following fields are defined in the response but are not used during service discovery.
+	//DefaultPassword string `json:"default_password,omitempty"`
+	//DateCreated     string `json:"date_created"`
+	//Status          string `json:"status"`
+	//PowerStatus     string `json:"power_status"`
+	//NetmaskV4       string `json:"netmask_v4"`
+	//GatewayV4       string `json:"gateway_v4"`
+	//V6Network       string `json:"v6_network"`
+	//V6NetworkSize   int    `json:"v6_network_size"`
+	//// Deprecated: Tag should no longer be used. Instead, use Tags.
+	//Tag             string `json:"tag"`
+	//KVM             string `json:"kvm"`
+	//AppID           int    `json:"app_id"`
+	//ImageID         string `json:"image_id"`
+	//FirewallGroupID string `json:"firewall_group_id"`
+	//UserScheme      string `json:"user_scheme"`
+}
+
+// Meta represents the available pagination information
+type Meta struct {
+	Total int `json:"total"`
+	Links *Links
+}
+
+// Links represent the next/previous cursor in your pagination calls
+type Links struct {
+	Next string `json:"next"`
+	Prev string `json:"prev"`
+}
+
+// getInstances retrieve instance from Vultr HTTP API.
+func getInstances(cfg *apiConfig) ([]Instance, error) {
+	var instances []Instance
+
+	// prepare GET params
+	params := url.Values{}
+	params.Set("per_page", "100")
+	params.Set("label", cfg.label)
+	params.Set("main_ip", cfg.mainIP)
+	params.Set("region", cfg.region)
+	params.Set("firewall_group_id", cfg.firewallGroupID)
+	params.Set("hostname", cfg.hostname)
+
+	// send request to vultr API
+	for {
+		// See: https://www.vultr.com/api/#tag/instances/operation/list-instances
+		path := fmt.Sprintf("/v2/instances?%s", params.Encode())
+		resp, err := cfg.c.GetAPIResponse(path)
+		if err != nil {
+			logger.Errorf("get response from vultr failed, path:%s, err: %v", path, err)
+			return nil, err
+		}
+
+		var listInstanceResp ListInstanceResponse
+		if err = json.Unmarshal(resp, &listInstanceResp); err != nil {
+			logger.Errorf("unmarshal response from vultr failed, err: %v", err)
+			return nil, err
+		}
+
+		instances = append(instances, listInstanceResp.Instances...)
+
+		if listInstanceResp.Meta != nil && listInstanceResp.Meta.Links != nil && listInstanceResp.Meta.Links.Next != "" {
+			// if `next page` is available, set the cursor param and request again.
+			params.Set("cursor", listInstanceResp.Meta.Links.Next)
+		} else {
+			// otherwise exit the loop
+			break
+		}
+	}
+
+	return instances, nil
+}
diff --git a/lib/promscrape/discovery/vultr/vultr.go b/lib/promscrape/discovery/vultr/vultr.go
new file mode 100644
index 000000000..d8999b2bb
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/vultr.go
@@ -0,0 +1,105 @@
+package vultr
+
+import (
+	"flag"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
+)
+
+const (
+	separator = ","
+)
+
+// SDCheckInterval defines interval for docker targets refresh.
+var SDCheckInterval = flag.Duration("promscrape.vultrSDCheckInterval", 30*time.Second, "Interval for checking for changes in Vultr. "+
+	"This works only if vultr_sd_configs is configured in '-promscrape.config' file. "+
+	"See https://docs.victoriametrics.com/sd_configs.html#vultr_sd_configs for details")
+
+// SDConfig represents service discovery config for Vultr.
+// See: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#vultr_sd_config
+// Additional query params are supported, while Prometheus only supports `Port` and HTTP auth.
+type SDConfig struct {
+	// API query params for filtering. All of them are optional.
+	// See: https://www.vultr.com/api/#tag/instances/operation/list-instances
+	Label           string `yaml:"label,omitempty"`
+	MainIP          string `yaml:"main_ip,omitempty"`
+	Region          string `yaml:"region,omitempty"`
+	FirewallGroupID string `yaml:"firewall_group_id,omitempty"`
+	Hostname        string `yaml:"hostname,omitempty"`
+
+	// The port to scrape metrics from. Default 80.
+	Port int `yaml:"port"`
+
+	// General HTTP / Auth configs.
+	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
+	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`
+	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
+
+	// refresh_interval is obtained from `-promscrape.vultrSDCheckInterval` command-line option.
+}
+
+var configMap = discoveryutils.NewConfigMap()
+
+// GetLabels returns gce labels according to sdc.
+func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
+	ac, err := getAPIConfig(sdc, baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get API config: %w", err)
+	}
+	instances, err := getInstances(ac)
+	if err != nil {
+		return nil, err
+	}
+	return getInstanceLabels(instances, ac.port), nil
+}
+
+// MustStop stops further usage for sdc.
+func (sdc *SDConfig) MustStop() {
+	configMap.Delete(sdc)
+}
+
+// getInstanceLabels returns labels for vultr instances obtained from the given cfg
+func getInstanceLabels(instances []Instance, port int) []*promutils.Labels {
+	ms := make([]*promutils.Labels, 0, len(instances))
+
+	for _, instance := range instances {
+		m := promutils.NewLabels(18)
+		m.Add("__address__", discoveryutils.JoinHostPort(instance.MainIP, port))
+		m.Add("__meta_vultr_instance_id", instance.ID)
+		m.Add("__meta_vultr_instance_label", instance.Label)
+		m.Add("__meta_vultr_instance_os", instance.Os)
+		m.Add("__meta_vultr_instance_os_id", strconv.Itoa(instance.OsID))
+		m.Add("__meta_vultr_instance_region", instance.Region)
+		m.Add("__meta_vultr_instance_plan", instance.Plan)
+		m.Add("__meta_vultr_instance_main_ip", instance.MainIP)
+		m.Add("__meta_vultr_instance_internal_ip", instance.InternalIP)
+		m.Add("__meta_vultr_instance_main_ipv6", instance.V6MainIP)
+		m.Add("__meta_vultr_instance_hostname", instance.Hostname)
+		m.Add("__meta_vultr_instance_server_status", instance.ServerStatus)
+		m.Add("__meta_vultr_instance_vcpu_count", strconv.Itoa(instance.VCPUCount))
+		m.Add("__meta_vultr_instance_ram_mb", strconv.Itoa(instance.RAM))
+		m.Add("__meta_vultr_instance_allowed_bandwidth_gb", strconv.Itoa(instance.AllowedBandwidth))
+		m.Add("__meta_vultr_instance_disk_gb", strconv.Itoa(instance.Disk))
+
+		// We surround the separated list with the separator as well. This way regular expressions
+		// in relabeling rules don't have to consider feature positions.
+		if len(instance.Features) > 0 {
+			features := separator + strings.Join(instance.Features, separator) + separator
+			m.Add("__meta_vultr_instance_features", features)
+		}
+
+		if len(instance.Tags) > 0 {
+			tags := separator + strings.Join(instance.Tags, separator) + separator
+			m.Add("__meta_vultr_instance_tags", tags)
+		}
+		ms = append(ms, m)
+	}
+	return ms
+}
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 797503298..d60511b8c 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -30,6 +30,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kuma"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/vultr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/metrics"
@@ -140,6 +141,7 @@ func runScraper(configFile string, pushData func(at *auth.Token, wr *prompbmarsh
 	scs.add("kuma_sd_configs", *kuma.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKumaSDScrapeWork(swsPrev) })
 	scs.add("nomad_sd_configs", *nomad.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getNomadSDScrapeWork(swsPrev) })
 	scs.add("openstack_sd_configs", *openstack.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
+	scs.add("vultr_sd_configs", *vultr.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getVultrSDScrapeWork(swsPrev) })
 	scs.add("yandexcloud_sd_configs", *yandexcloud.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getYandexCloudSDScrapeWork(swsPrev) })
 	scs.add("static_configs", 0, func(cfg *Config, _ []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })
 


Test patch:
diff --git a/lib/promscrape/discovery/vultr/api_test.go b/lib/promscrape/discovery/vultr/api_test.go
new file mode 100644
index 000000000..06e491446
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/api_test.go
@@ -0,0 +1,16 @@
+package vultr
+
+import (
+	"testing"
+)
+
+func TestNewAPIConfig(t *testing.T) {
+
+	sdc := &SDConfig{}
+	baseDir := "."
+	_, err := newAPIConfig(sdc, baseDir)
+	if err != nil {
+		t.Errorf("newAPIConfig failed with, err: %v", err)
+		return
+	}
+}
diff --git a/lib/promscrape/discovery/vultr/instance_test.go b/lib/promscrape/discovery/vultr/instance_test.go
new file mode 100644
index 000000000..b888c1a13
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/instance_test.go
@@ -0,0 +1,301 @@
+package vultr
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+// TestGetInstances runs general test cases for GetInstances
+func TestGetInstances(t *testing.T) {
+	testCases := []struct {
+		name           string
+		apiResponse    string
+		apiError       bool
+		expectError    bool
+		expectResponse []Instance
+	}{
+		{
+			name:           "success response",
+			apiResponse:    mockListInstanceSuccessResp,
+			apiError:       false,
+			expectError:    false,
+			expectResponse: expectSuccessInstances,
+		},
+		{
+			name:           "failed response",
+			apiResponse:    mockListInstanceFailedResp,
+			apiError:       true,
+			expectError:    true,
+			expectResponse: nil,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			// Prepare a mock Vultr server.
+			mockServer := newMockVultrServer(func() ([]byte, error) {
+				var e error
+				if tt.apiError {
+					e = errors.New("mock error")
+				}
+				return []byte(tt.apiResponse), e
+			})
+
+			// Prepare a discovery HTTP client who calls mock server.
+			client, _ := discoveryutils.NewClient(mockServer.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+			cfg := &apiConfig{
+				c: client,
+			}
+
+			// execute `getInstances`
+			instances, err := getInstances(cfg)
+
+			// evaluate test result
+			if tt.expectError != (err != nil) {
+				t.Errorf("getInstances expect (error != nil): %t, got error: %v", tt.expectError, err)
+			}
+
+			if !reflect.DeepEqual(tt.expectResponse, instances) {
+				t.Errorf("getInstances expect result: %v, got: %v", tt.expectResponse, instances)
+			}
+		})
+	}
+}
+
+// TestGetInstancesPaging run test cases for response with multiple pages.
+func TestGetInstancesPaging(t *testing.T) {
+	// Prepare a mock Vultr server.
+	// requestCount control the mock response for different page request.
+	requestCount := 0
+
+	mockServer := newMockVultrServer(func() ([]byte, error) {
+		// for the 1st request, response with `next` cursor
+		if requestCount == 0 {
+			requestCount++
+			return []byte(mockListInstanceSuccessPage0Resp), nil
+		}
+		// for the 2nd+ request, response with `prev` cursor and empty `next`.
+		return []byte(mockListInstanceSuccessPage1Resp), nil
+	})
+
+	// Prepare a discovery HTTP client who calls mock server.
+	client, _ := discoveryutils.NewClient(mockServer.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	cfg := &apiConfig{
+		c: client,
+	}
+
+	// execute `getInstances`
+	instances, err := getInstances(cfg)
+
+	// evaluate test result
+	if err != nil {
+		t.Errorf("getInstances expect error: %v, got error: %v", nil, err)
+	}
+
+	if !reflect.DeepEqual(expectSuccessPagingInstances, instances) {
+		t.Errorf("getInstances expect result: %v, got: %v", expectSuccessPagingInstances, instances)
+	}
+}
+
+// ------------ Test dataset ------------
+var (
+	// mockListInstanceSuccessResp is crawled from a real-world response of ListInstance API
+	// with sensitive info removed/modified.
+	mockListInstanceSuccessResp = `{
+	"instances": [{
+		"id": "fake-id-07f7-4b68-88ac-fake-id",
+		"os": "Ubuntu 22.04 x64",
+		"ram": 1024,
+		"disk": 25,
+		"main_ip": "64.176.84.27",
+		"vcpu_count": 1,
+		"region": "sgp",
+		"plan": "vc2-1c-1gb",
+		"date_created": "2024-04-05T05:41:28+00:00",
+		"status": "active",
+		"allowed_bandwidth": 1,
+		"netmask_v4": "255.255.254.0",
+		"gateway_v4": "64.176.63.2",
+		"power_status": "running",
+		"server_status": "installingbooting",
+		"v6_network": "2002:18f0:4100:263a::",
+		"v6_main_ip": "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+		"v6_network_size": 64,
+		"label": "vultr-sd",
+		"internal_ip": "",
+		"kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=secret_data_string",
+		"hostname": "vultr-sd",
+		"tag": "",
+		"tags": [],
+		"os_id": 1743,
+		"app_id": 0,
+		"image_id": "",
+		"firewall_group_id": "",
+		"features": ["ipv6"],
+		"user_scheme": "root"
+	}],
+	"meta": {
+		"total": 1,
+		"links": {
+			"next": "",
+			"prev": ""
+		}
+	}
+}`
+	expectSuccessInstances = []Instance{
+		{
+			ID:               "fake-id-07f7-4b68-88ac-fake-id",
+			Os:               "Ubuntu 22.04 x64",
+			RAM:              1024,
+			Disk:             25,
+			MainIP:           "64.176.84.27",
+			VCPUCount:        1,
+			Region:           "sgp",
+			Plan:             "vc2-1c-1gb",
+			AllowedBandwidth: 1,
+			ServerStatus:     "installingbooting",
+			V6MainIP:         "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			Label:            "vultr-sd",
+			InternalIP:       "",
+			Hostname:         "vultr-sd",
+			Tags:             []string{},
+			OsID:             1743,
+			Features:         []string{"ipv6"},
+		},
+	}
+)
+
+var (
+	mockListInstanceFailedResp = `{"error":"Invalid API token.","status":401}`
+)
+
+var (
+	// mockListInstanceSuccessPage0Resp contains `next` cursor
+	mockListInstanceSuccessPage0Resp = `{
+	"instances": [{
+		"id": "fake-id-07f7-4b68-88ac-fake-id",
+		"os": "Ubuntu 22.04 x64",
+		"ram": 1024,
+		"disk": 25,
+		"main_ip": "64.176.84.27",
+		"vcpu_count": 1,
+		"region": "sgp",
+		"plan": "vc2-1c-1gb",
+		"date_created": "2024-04-05T05:41:28+00:00",
+		"status": "active",
+		"allowed_bandwidth": 1,
+		"netmask_v4": "255.255.254.0",
+		"gateway_v4": "64.176.63.2",
+		"power_status": "running",
+		"server_status": "installingbooting",
+		"v6_network": "2002:18f0:4100:263a::",
+		"v6_main_ip": "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+		"v6_network_size": 64,
+		"label": "vultr-sd",
+		"internal_ip": "",
+		"kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=secret_data_string",
+		"hostname": "vultr-sd",
+		"tag": "",
+		"tags": [],
+		"os_id": 1743,
+		"app_id": 0,
+		"image_id": "",
+		"firewall_group_id": "",
+		"features": ["ipv6"],
+		"user_scheme": "root"
+	}],
+	"meta": {
+		"total": 2,
+		"links": {
+			"next": "fake-cursor-string",
+			"prev": ""
+		}
+	}
+}`
+	// mockListInstanceSuccessPage1Resp contains `prev` cursor
+	mockListInstanceSuccessPage1Resp = `{
+	"instances": [{
+		"id": "fake-id-07f7-4b68-88ac-fake-id",
+		"os": "Ubuntu 22.04 x64",
+		"ram": 1024,
+		"disk": 25,
+		"main_ip": "64.176.84.27",
+		"vcpu_count": 1,
+		"region": "sgp",
+		"plan": "vc2-1c-1gb",
+		"date_created": "2024-04-05T05:41:28+00:00",
+		"status": "active",
+		"allowed_bandwidth": 1,
+		"netmask_v4": "255.255.254.0",
+		"gateway_v4": "64.176.63.2",
+		"power_status": "running",
+		"server_status": "installingbooting",
+		"v6_network": "2002:18f0:4100:263a::",
+		"v6_main_ip": "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+		"v6_network_size": 64,
+		"label": "vultr-sd",
+		"internal_ip": "",
+		"kvm": "https:\/\/my.vultr.com\/subs\/vps\/novnc\/api.php?data=secret_data_string",
+		"hostname": "vultr-sd",
+		"tag": "",
+		"tags": [],
+		"os_id": 1743,
+		"app_id": 0,
+		"image_id": "",
+		"firewall_group_id": "",
+		"features": ["ipv6"],
+		"user_scheme": "root"
+	}],
+	"meta": {
+		"total": 2,
+		"links": {
+			"next": "",
+			"prev": "fake-cursor-string"
+		}
+	}
+}`
+	expectSuccessPagingInstances = []Instance{
+		{
+			ID:               "fake-id-07f7-4b68-88ac-fake-id",
+			Os:               "Ubuntu 22.04 x64",
+			RAM:              1024,
+			Disk:             25,
+			MainIP:           "64.176.84.27",
+			VCPUCount:        1,
+			Region:           "sgp",
+			Plan:             "vc2-1c-1gb",
+			AllowedBandwidth: 1,
+			ServerStatus:     "installingbooting",
+			V6MainIP:         "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			Label:            "vultr-sd",
+			InternalIP:       "",
+			Hostname:         "vultr-sd",
+			Tags:             []string{},
+			OsID:             1743,
+			Features:         []string{"ipv6"},
+		},
+		{
+			ID:               "fake-id-07f7-4b68-88ac-fake-id",
+			Os:               "Ubuntu 22.04 x64",
+			RAM:              1024,
+			Disk:             25,
+			MainIP:           "64.176.84.27",
+			VCPUCount:        1,
+			Region:           "sgp",
+			Plan:             "vc2-1c-1gb",
+			AllowedBandwidth: 1,
+			ServerStatus:     "installingbooting",
+			V6MainIP:         "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			Label:            "vultr-sd",
+			InternalIP:       "",
+			Hostname:         "vultr-sd",
+			Tags:             []string{},
+			OsID:             1743,
+			Features:         []string{"ipv6"},
+		},
+	}
+)
diff --git a/lib/promscrape/discovery/vultr/mock_server_test.go b/lib/promscrape/discovery/vultr/mock_server_test.go
new file mode 100644
index 000000000..c0dd79e46
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/mock_server_test.go
@@ -0,0 +1,40 @@
+package vultr
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+)
+
+func newMockVultrServer(jsonResponse func() ([]byte, error)) *vultrServer {
+	rw := &vultrServer{}
+	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
+	rw.jsonResponse = jsonResponse
+	return rw
+}
+
+type vultrServer struct {
+	*httptest.Server
+	jsonResponse func() ([]byte, error)
+}
+
+func (rw *vultrServer) err(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusBadRequest)
+	w.Write([]byte(err.Error()))
+}
+
+func (rw *vultrServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		rw.err(w, fmt.Errorf("bad method %q", r.Method))
+		return
+	}
+
+	resp, err := rw.jsonResponse()
+	if err != nil {
+		rw.err(w, err)
+		return
+	}
+
+	w.Write(resp)
+	w.WriteHeader(http.StatusOK)
+}
diff --git a/lib/promscrape/discovery/vultr/vultr_test.go b/lib/promscrape/discovery/vultr/vultr_test.go
new file mode 100644
index 000000000..572782726
--- /dev/null
+++ b/lib/promscrape/discovery/vultr/vultr_test.go
@@ -0,0 +1,96 @@
+package vultr
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestGetInstanceLabels(t *testing.T) {
+	input := []Instance{
+		{
+			ID:               "fake-id-07f7-4b68-88ac-fake-id",
+			Os:               "Ubuntu 22.04 x64",
+			RAM:              1024,
+			Disk:             25,
+			MainIP:           "64.176.84.27",
+			VCPUCount:        1,
+			Region:           "sgp",
+			Plan:             "vc2-1c-1gb",
+			AllowedBandwidth: 1,
+			ServerStatus:     "installingbooting",
+			V6MainIP:         "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			Label:            "vultr-sd",
+			InternalIP:       "",
+			Hostname:         "vultr-sd",
+			Tags:             []string{"mock tags"},
+			OsID:             1743,
+			Features:         []string{"ipv6"},
+		},
+		{
+			ID:               "fake-id-07f7-4b68-88ac-fake-id",
+			Os:               "Ubuntu 22.04 x64",
+			RAM:              1024,
+			Disk:             25,
+			MainIP:           "64.176.84.27",
+			VCPUCount:        1,
+			Region:           "sgp",
+			Plan:             "vc2-1c-1gb",
+			AllowedBandwidth: 1,
+			ServerStatus:     "installingbooting",
+			V6MainIP:         "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			Label:            "vultr-sd",
+			InternalIP:       "",
+			Hostname:         "vultr-sd",
+			Tags:             []string{"mock tags"},
+			OsID:             1743,
+			Features:         []string{"ipv6"},
+		},
+	}
+
+	expect := []*promutils.Labels{
+		promutils.NewLabelsFromMap(map[string]string{
+			"__address__":                                "64.176.84.27:8080",
+			"__meta_vultr_instance_id":                   "fake-id-07f7-4b68-88ac-fake-id",
+			"__meta_vultr_instance_label":                "vultr-sd",
+			"__meta_vultr_instance_os":                   "Ubuntu 22.04 x64",
+			"__meta_vultr_instance_os_id":                "1743",
+			"__meta_vultr_instance_region":               "sgp",
+			"__meta_vultr_instance_plan":                 "vc2-1c-1gb",
+			"__meta_vultr_instance_main_ip":              "64.176.84.27",
+			"__meta_vultr_instance_internal_ip":          "",
+			"__meta_vultr_instance_main_ipv6":            "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			"__meta_vultr_instance_hostname":             "vultr-sd",
+			"__meta_vultr_instance_server_status":        "installingbooting",
+			"__meta_vultr_instance_vcpu_count":           "1",
+			"__meta_vultr_instance_ram_mb":               "1024",
+			"__meta_vultr_instance_allowed_bandwidth_gb": "1",
+			"__meta_vultr_instance_disk_gb":              "25",
+			"__meta_vultr_instance_features":             ",ipv6,",
+			"__meta_vultr_instance_tags":                 ",mock tags,",
+		}),
+		promutils.NewLabelsFromMap(map[string]string{
+			"__address__":                                "64.176.84.27:8080",
+			"__meta_vultr_instance_id":                   "fake-id-07f7-4b68-88ac-fake-id",
+			"__meta_vultr_instance_label":                "vultr-sd",
+			"__meta_vultr_instance_os":                   "Ubuntu 22.04 x64",
+			"__meta_vultr_instance_os_id":                "1743",
+			"__meta_vultr_instance_region":               "sgp",
+			"__meta_vultr_instance_plan":                 "vc2-1c-1gb",
+			"__meta_vultr_instance_main_ip":              "64.176.84.27",
+			"__meta_vultr_instance_internal_ip":          "",
+			"__meta_vultr_instance_main_ipv6":            "2002:18f0:4100:263a:5300:07ff:fdd7:691c",
+			"__meta_vultr_instance_hostname":             "vultr-sd",
+			"__meta_vultr_instance_server_status":        "installingbooting",
+			"__meta_vultr_instance_vcpu_count":           "1",
+			"__meta_vultr_instance_ram_mb":               "1024",
+			"__meta_vultr_instance_allowed_bandwidth_gb": "1",
+			"__meta_vultr_instance_disk_gb":              "25",
+			"__meta_vultr_instance_features":             ",ipv6,",
+			"__meta_vultr_instance_tags":                 ",mock tags,",
+		}),
+	}
+	labels := getInstanceLabels(input, 8080)
+	discoveryutils.TestEqualLabelss(t, labels, expect)
+}