Repository: VictoriaMetrics/VictoriaMetrics
Pull request #3194 (2022-10-04 12:25:50)

vmalert: allow using `{{$labels}}` for templating in `-external.alert.source`
The change supposed to provide additional flexibility for generating alert's source link based on labels values.

Signed-off-by: hagen1778 <roman@victoriametrics.com>


Patch:
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 46d846f39..0e31d9854 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -264,7 +264,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, nil, m)
+		templated, err := alert.ExecTemplate(nil, alert.Labels, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
diff --git a/deployment/docker/alerts-cluster.yml b/deployment/docker/alerts-cluster.yml
index 5f2587cf2..c4918c2a1 100644
--- a/deployment/docker/alerts-cluster.yml
+++ b/deployment/docker/alerts-cluster.yml
@@ -89,17 +89,6 @@ groups:
           description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n
             When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU."
 
-      - alert: TooManyLogs
-        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) by (job, instance) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=104&var-instance={{ $labels.instance }}"
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
-          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
-           Worth to check logs for specific error messages."
-
       - alert: RowsRejectedOnIngestion
         expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
         for: 15m
diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/alerts-health.yml
index 489a7035f..4668718e6 100644
--- a/deployment/docker/alerts-health.yml
+++ b/deployment/docker/alerts-health.yml
@@ -51,4 +51,14 @@ groups:
         annotations:
           summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
           description: "Too high CPU usage may be a sign of insufficient resources and make process unstable.
-               Consider to either increase available CPU resources or decrease the load on the process."
\ No newline at end of file
+               Consider to either increase available CPU resources or decrease the load on the process."
+
+      - alert: TooManyLogs
+        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) by (job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
+         Worth to check logs for specific error messages."
diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml
index 7cf31ff8a..bdcf625ec 100644
--- a/deployment/docker/alerts.yml
+++ b/deployment/docker/alerts.yml
@@ -71,17 +71,6 @@ groups:
           description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n
             When VictoriaMetrics constantly hits the limit it means that storage is overloaded and requires more CPU."
 
-      - alert: TooManyLogs
-        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) by (job, instance) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=67&var-instance={{ $labels.instance }}"
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
-          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
-           Worth to check logs for specific error messages."
-
       - alert: RowsRejectedOnIngestion
         expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
         for: 15m
@@ -154,4 +143,4 @@ groups:
           summary: "Metrics ingested in ({{ $labels.instance }}) are exceeding labels limit"
           description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
            This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
-           correctly or that clients which send these metrics aren't misbehaving."
\ No newline at end of file
+           correctly or that clients which send these metrics aren't misbehaving."
diff --git a/docs/Articles.md b/docs/Articles.md
index ff9e26c4e..61996462a 100644
--- a/docs/Articles.md
+++ b/docs/Articles.md
@@ -104,6 +104,7 @@ See also [case studies](https://docs.victoriametrics.com/CaseStudies.html).
 * [How to use relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
 * [How to monitor Go applications with VictoriaMetrics](https://victoriametrics.medium.com/how-to-monitor-go-applications-with-victoriametrics-c04703110870)
 * [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
+* [Cardinality explorer](https://victoriametrics.com/blog/cardinality-explorer/)
 
 ### Other articles
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f828e66bc..c68102047 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -41,6 +41,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add experimental feature for displaying last 10 states of the rule (recording or alerting) evaluation. The state is available on the Rule page, which can be opened by clicking on `Details` link next to Rule's name on the `/groups` page.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using extra labels in annotiations. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3013).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring authorization params per list of targets in vmalert's notifier config for `static_configs`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2690).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using `{{$labels}}` for templating in command-line flag `-external.alert.source`. The change supposed to provide additional flexibility for generating alert's source link based on labels values.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): minimize the time needed for reading large responses from scrape targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). This should reduce scrape durations for such targets as [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) running in a big Kubernetes cluster.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [sort_by_label_numeric](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric) and [sort_by_label_numeric_desc](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric_desc) functions for [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html) of input time series by the specified labels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2938).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html) and [vmrestore](https://docs.victoriametrics.com/vmrestore.html): retry GCS operations for up to 3 minutes on temporary failures. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3147).
diff --git a/docs/README.md b/docs/README.md
index 9a2539643..907d6cd94 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -311,7 +311,7 @@ matching the specified [series selector](https://prometheus.io/docs/prometheus/l
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
-
+See the example of using the cardinality explorer [here](https://victoriametrics.com/blog/cardinality-explorer/).
 
 ## How to apply new config to VictoriaMetrics
 
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index da3e76d27..026899c59 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -315,7 +315,7 @@ matching the specified [series selector](https://prometheus.io/docs/prometheus/l
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
-
+See the example of using the cardinality explorer [here](https://victoriametrics.com/blog/cardinality-explorer/).
 
 ## How to apply new config to VictoriaMetrics
 


Test patch:
diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go
index da21702b9..ce495df4c 100644
--- a/app/vmalert/main_test.go
+++ b/app/vmalert/main_test.go
@@ -34,7 +34,7 @@ func TestGetExternalURL(t *testing.T) {
 }
 
 func TestGetAlertURLGenerator(t *testing.T) {
-	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4}
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
@@ -48,11 +48,11 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	if err == nil {
 		t.Errorf("expected template validation error got nil")
 	}
-	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true)
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/foo?query=4"; exp != fn(testAlert) {
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }