Repository: VictoriaMetrics/VictoriaMetrics
Pull request #2457 (2022-04-12 16:36:14)

lib/promscrape: removes omitempty for ScrapeConfig
This change fixes incorrect marshalling for ScrapeConfig
it affects http endpoint and ScrapeConfig checksum.

With omitempty, custom Marshaller is not called if field is not a pointer.

Previously this issue happened at vmalert


Patch:
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index ed6692c1c..63376fe90 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -33,7 +33,7 @@ of Grafana dashboards if possible:
 
 See how to setup monitoring here:
 * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-* [montioring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+* [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
 
 **Version**
 The line returned when passing `--version` command line flag to the binary. For example:
diff --git a/app/vmalert/Makefile b/app/vmalert/Makefile
index 87f08e6a6..eebac7f01 100644
--- a/app/vmalert/Makefile
+++ b/app/vmalert/Makefile
@@ -83,7 +83,7 @@ run-vmalert-sd: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
-		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-notifier.config=app/vmalert/notifier/testdata/mixed.good.yaml \
 		-configCheckInterval=10s
 
 replay-vmalert: vmalert
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index b03ef76dd..185a9d891 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -48,7 +48,7 @@ To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
 * datasource address - reachable MetricsQL endpoint to run queries against;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via
+aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
 [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
   compatible storage to persist rules and alerts state info;
@@ -846,8 +846,9 @@ Notifier also supports configuration via file specified with flag `notifier.conf
   -notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
 ```
 
-The configuration file allows to configure static notifiers or discover notifiers via
-[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config).
+The configuration file allows to configure static notifiers, discover notifiers via
+[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
+and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config):
 For example:
 
 ```
@@ -860,6 +861,12 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - alertmanager
+      
+dns_sd_configs:
+  - names:
+      - my.domain.com
+    type: 'A'
+    port: 9093
 ```
 
 The list of configured or discovered Notifiers can be explored via [UI](#Web).
@@ -911,6 +918,11 @@ static_configs:
 consul_sd_configs:
   [ - <consul_sd_config> ... ]
 
+# List of DNS service discovery configurations.
+# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+dns_sd_configs:
+  [ - <dns_sd_config> ... ]
+
 # List of relabel configurations for entities discovered via service discovery.
 # Supports the same relabeling features as the rest of VictoriaMetrics components.
 # See https://docs.victoriametrics.com/vmagent.html#relabeling
diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go
index c6e75aef8..3eb776fbb 100644
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -29,6 +30,10 @@ type Config struct {
 	// ConsulSDConfigs contains list of settings for service discovery via Consul
 	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// DNSSDConfigs ontains list of settings for service discovery via DNS.
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
+
 	// StaticConfigs contains list of static targets
 	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
 
diff --git a/app/vmalert/notifier/config_watcher.go b/app/vmalert/notifier/config_watcher.go
index 0fe8c53f5..1bc116229 100644
--- a/app/vmalert/notifier/config_watcher.go
+++ b/app/vmalert/notifier/config_watcher.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
 
 // configWatcher supports dynamic reload of Notifier objects
@@ -195,6 +196,24 @@ func (cw *configWatcher) start() error {
 			return fmt.Errorf("failed to start consulSD discovery: %s", err)
 		}
 	}
+
+	if len(cw.cfg.DNSSDConfigs) > 0 {
+		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.DNSSDConfigs {
+				sdc := &cw.cfg.DNSSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start DNSSD discovery: %s", err)
+		}
+	}
 	return nil
 }
 
diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go
index b00c1bcd6..99892d2b4 100644
--- a/app/vmalert/notifier/init.go
+++ b/app/vmalert/notifier/init.go
@@ -162,6 +162,8 @@ const (
 	TargetStatic TargetType = "static"
 	// TargetConsul is for targets discovered via Consul
 	TargetConsul TargetType = "consulSD"
+	// TargetDNS is for targets discovered via DNS
+	TargetDNS TargetType = "DNSSD"
 )
 
 // GetTargets returns list of static or discovered targets
diff --git a/app/vmctl/main.go b/app/vmctl/main.go
index 52176ade0..abf5df6af 100644
--- a/app/vmctl/main.go
+++ b/app/vmctl/main.go
@@ -215,7 +215,7 @@ func main() {
 
 	err = app.Run(os.Args)
 	if err != nil {
-		log.Println(err)
+		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
 }
diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index 6699bc81e..a4eceaa5c 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -4,7 +4,7 @@ DOCKER_NAMESPACE := victoriametrics
 
 ROOT_IMAGE ?= alpine:3.15.4
 CERTS_IMAGE := alpine:3.15.4
-GO_BUILDER_IMAGE := golang:1.18.0-alpine
+GO_BUILDER_IMAGE := golang:1.18.1-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.3-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 69f7f03c9..e87e7eabf 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,10 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 ## tip
 
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for DNS-based discovery for notifiers in the same way as Prometheus does. See [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2460).
+
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): return non-zero exit code on error. This allows handling `vmctl` errors in shell scripts. Previously `vmctl` was returning 0 exit code on error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2322).
+
 
 ## [v1.76.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.76.1)
 
@@ -39,7 +43,7 @@ Released at 07-04-2022
 **Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics, so `vmselect` and `vmstorage` nodes may log communication errors during the upgrade. These errors should stop after all the `vmselect` and `vmstorage` nodes are updated to new release.
 
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add ability to verify files obtained via [native export](https://docs.victoriametrics.com/#how-to-export-data-in-native-format). See [these docs](https://docs.victoriametrics.com/vmctl.html#verifying-exported-blocks-from-victoriametrics) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2362).
-* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add pre-defined dasbhoards for per-job CPU usage, memory usage and disk IO usage. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2243) for details.
+* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add pre-defined dashboards for per-job CPU usage, memory usage and disk IO usage. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2243) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): improve compatibility with [Prometheus Alert Generator specification](https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2340).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `-datasource.disableKeepAlive` command-line flag, which can be used for disabling [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) to datasources. This option can be useful for distributing load among multiple datasources behind TCP proxy such as [HAProxy](http://www.haproxy.org/).
 * FEATURE: [Cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce memory usage by up to 50% for `vminsert` and `vmstorage` under high ingestion rate.
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index 78e84b157..77187411e 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -129,8 +129,9 @@ A minimal cluster must contain the following nodes:
 - a single `vminsert` node with `-storageNode=<vmstorage_host>`
 - a single `vmselect` node with `-storageNode=<vmstorage_host>`
 
-It is recommended to run at least two nodes for each service
-for high availability purposes.
+It is recommended to run at least two nodes for each service for high availability purposes. In this case the cluster continues working when a single node is temporarily unavailable and the remaining nodes can handle the increased workload. The node may be temporarily unavailable when the underlying hardware breaks, during software upgrades, migration or other maintenance tasks.
+
+It is preferred to run many small `vmstorage` nodes over a few big `vmstorage` nodes, since this reduces the workload increase on the remaining `vmstorage` nodes when some of `vmstorage` nodes become temporarily unavailable.
 
 An http load balancer such as [vmauth](https://docs.victoriametrics.com/vmauth.html) or `nginx` must be put in front of `vminsert` and `vmselect` nodes. It must contain the following routing configs according to [the url format](#url-format):
 
@@ -255,15 +256,18 @@ It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.co
 
 ## Cluster resizing and scalability
 
-Cluster performance and capacity scales with adding new nodes.
+Cluster performance and capacity can be scaled up in two ways:
+
+- By adding more resources (CPU, RAM, disk IO, disk space, network bandwidth) to existing nodes in the cluster (aka vertical scalability).
+- By adding more nodes to the cluster (aka horizontal scalability).
+
+General recommendations for cluster scalability:
 
-- `vminsert` and `vmselect` nodes are stateless and may be added / removed at any time.
-  Do not forget updating the list of these nodes on http load balancer.
-  Adding more `vminsert` nodes scales data ingestion rate. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/175#issuecomment-536925841)
-  about ingestion rate scalability.
-  Adding more `vmselect` nodes scales select queries rate.
-- `vmstorage` nodes own the ingested data, so they cannot be removed without data loss.
-  Adding more `vmstorage` nodes scales cluster capacity.
+- Adding more CPU and RAM to existing `vmselect` nodes improves the performance for heavy queries, which process big number of time series with big number of raw samples.
+- Adding more `vmstorage` nodes increases the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series) the cluster can handle. This also increases query performance over time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). The cluster stability is also improved with the number of `vmstorage` nodes, since active `vmstorage` nodes need to handle lower additional workload when some of `vmstorage` nodes become unavailable.
+- Adding more CPU and RAM to existing `vmstorage` nodes increases the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series) the cluster can handle. It is preferred to add more `vmstorage` nodes over adding more CPU and RAM to existing `vmstorage` nodes, since higher number of `vmstorage` nodes increases cluster stability and improves query performance over time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
+- Adding more `vminsert` nodes increases the maximum possible data ingestion speed, since the ingested data may be split among bigger number of `vminsert` nodes.
+- Adding more `vmselect` nodes increases the maximum possible queries rate, since the incoming concurrent requests may be split among bigger number of `vmselect` nodes.
 
 Steps to add `vmstorage` node:
 
@@ -336,7 +340,12 @@ Then [promxy](https://github.com/jacksontj/promxy) could be used for querying th
 
 `vminsert` nodes can accept data from another `vminsert` nodes starting from [v1.60.0](https://docs.victoriametrics.com/CHANGELOG.html#v1600) if `-clusternativeListenAddr` command-line flag is set. For example, if `vminsert` is started with `-clusternativeListenAddr=:8400` command-line flag, then it can accept data from another `vminsert` nodes at TCP port 8400 in the same way as `vmstorage` nodes do. This allows chaining `vminsert` nodes and building multi-level cluster topologies with flexible configs. For example, the top level of `vminsert` nodes can replicate data among the second level of `vminsert` nodes located in distinct availability zones (AZ), while the second-level `vminsert` nodes can spread the data among `vmstorage` nodes located in the same AZ. Such setup guarantees cluster availability if some AZ becomes unavailable. The data from all the `vmstorage` nodes in all the AZs can be read via `vmselect` nodes, which are configured to query all the `vmstorage` nodes in all the availability zones (e.g. all the `vmstorage` addresses are passed via `-storageNode` command-line flag to `vmselect` nodes). Additionally, `-replicationFactor=k+1` must be passed to `vmselect` nodes, where `k` is the lowest number of `vmstorage` nodes in a single AZ. See [replication docs](#replication-and-data-safety) for more details.
 
-Another option is to set up [vmagent](https://docs.victoriametrics.com/vmagent.html) for replicating the data among multiple VictoriaMetrics clusters. See [these docs](https://docs.victoriametrics.com/vmagent.html#multitenancy) for details.
+The multi-level cluster setup for `vminsert` nodes has the following shortcomings because of synchronous replication and data sharding:
+
+* Data ingestion speed is limited by the slowest link to AZ.
+* `vminsert` nodes at top level re-route incoming data to the remaining AZs when some AZs are temporariliy unavailable. This results in data gaps at AZs which were temporarily unavailable.
+
+These issues are addressed by [vmagent](https://docs.victoriametrics.com/vmagent.html) when it runs in [multitenancy mode](https://docs.victoriametrics.com/vmagent.html#multitenancy). `vmagent` buffers data, which must be sent to a particular AZ, when this AZ is temporarily unavailable. The buffer is stored on disk. The buffered data is sent to AZ as soon as it becomes available.
 
 ## Helm
 
diff --git a/docs/vmalert.md b/docs/vmalert.md
index 0307f4f6e..5150ae324 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -52,7 +52,7 @@ To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
 * datasource address - reachable MetricsQL endpoint to run queries against;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via
+aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
 [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
   compatible storage to persist rules and alerts state info;
@@ -850,8 +850,9 @@ Notifier also supports configuration via file specified with flag `notifier.conf
   -notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
 ```
 
-The configuration file allows to configure static notifiers or discover notifiers via
-[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config).
+The configuration file allows to configure static notifiers, discover notifiers via
+[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
+and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config):
 For example:
 
 ```
@@ -864,6 +865,12 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - alertmanager
+      
+dns_sd_configs:
+  - names:
+      - my.domain.com
+    type: 'A'
+    port: 9093
 ```
 
 The list of configured or discovered Notifiers can be explored via [UI](#Web).
@@ -915,6 +922,11 @@ static_configs:
 consul_sd_configs:
   [ - <consul_sd_config> ... ]
 
+# List of DNS service discovery configurations.
+# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+dns_sd_configs:
+  [ - <dns_sd_config> ... ]
+
 # List of relabel configurations for entities discovered via service discovery.
 # Supports the same relabeling features as the rest of VictoriaMetrics components.
 # See https://docs.victoriametrics.com/vmagent.html#relabeling
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 15e2feef6..bca411b2f 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -134,8 +134,8 @@ type GlobalConfig struct {
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
 type ScrapeConfig struct {
 	JobName              string                      `yaml:"job_name"`
-	ScrapeInterval       promutils.Duration          `yaml:"scrape_interval,omitempty"`
-	ScrapeTimeout        promutils.Duration          `yaml:"scrape_timeout,omitempty"`
+	ScrapeInterval       promutils.Duration          `yaml:"scrape_interval"`
+	ScrapeTimeout        promutils.Duration          `yaml:"scrape_timeout"`
 	MetricsPath          string                      `yaml:"metrics_path,omitempty"`
 	HonorLabels          bool                        `yaml:"honor_labels,omitempty"`
 	HonorTimestamps      *bool                       `yaml:"honor_timestamps,omitempty"`
@@ -168,8 +168,8 @@ type ScrapeConfig struct {
 	DisableCompression  bool                       `yaml:"disable_compression,omitempty"`
 	DisableKeepAlive    bool                       `yaml:"disable_keepalive,omitempty"`
 	StreamParse         bool                       `yaml:"stream_parse,omitempty"`
-	ScrapeAlignInterval promutils.Duration         `yaml:"scrape_align_interval,omitempty"`
-	ScrapeOffset        promutils.Duration         `yaml:"scrape_offset,omitempty"`
+	ScrapeAlignInterval promutils.Duration         `yaml:"scrape_align_interval"`
+	ScrapeOffset        promutils.Duration         `yaml:"scrape_offset"`
 	SeriesLimit         int                        `yaml:"series_limit,omitempty"`
 	ProxyClientConfig   promauth.ProxyClientConfig `yaml:",inline"`
 
diff --git a/snap/local/Makefile b/snap/local/Makefile
index ef9146652..ba264a798 100644
--- a/snap/local/Makefile
+++ b/snap/local/Makefile
@@ -1,4 +1,4 @@
-GO_VERSION ?=1.17.7
+GO_VERSION ?=1.18.1
 SNAP_BUILDER_IMAGE := local/snap-builder:2.0.0-$(shell echo $(GO_VERSION) | tr :/ __)
 
 


Test patch:
diff --git a/app/vmalert/notifier/config_test.go b/app/vmalert/notifier/config_test.go
index e3dfe6b50..442b2f3a7 100644
--- a/app/vmalert/notifier/config_test.go
+++ b/app/vmalert/notifier/config_test.go
@@ -12,6 +12,7 @@ func TestConfigParseGood(t *testing.T) {
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
+	f("testdata/dns.good.yaml")
 	f("testdata/static.good.yaml")
 }
 
diff --git a/app/vmalert/notifier/testdata/dns.good.yaml b/app/vmalert/notifier/testdata/dns.good.yaml
new file mode 100644
index 000000000..e7a50c472
--- /dev/null
+++ b/app/vmalert/notifier/testdata/dns.good.yaml
@@ -0,0 +1,12 @@
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+relabel_configs:
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"
\ No newline at end of file
diff --git a/app/vmalert/notifier/testdata/mixed.good.yaml b/app/vmalert/notifier/testdata/mixed.good.yaml
index f6a5c5a62..906c2aa6a 100644
--- a/app/vmalert/notifier/testdata/mixed.good.yaml
+++ b/app/vmalert/notifier/testdata/mixed.good.yaml
@@ -11,8 +11,18 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - consul
+
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+
 relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
\ No newline at end of file
+    target_label: __scheme__
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name
\ No newline at end of file