Repository: VictoriaMetrics/VictoriaMetrics
Pull request #2198 (2022-02-15 14:55:06)

dashboards: update the threshold for slow inserts % on the dashboard
Signed-off-by: hagen1778 <roman@victoriametrics.com>

------------

Mirrored change from master branch https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2197


Patch:
diff --git a/README.md b/README.md
index 33e1a6c0b..665945c31 100644
--- a/README.md
+++ b/README.md
@@ -586,9 +586,9 @@ Below is the output for `/path/to/vminsert -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
@@ -712,9 +712,9 @@ Below is the output for `/path/to/vmselect -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
@@ -815,9 +815,9 @@ Below is the output for `/path/to/vmstorage -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
   -vminsertAddr string
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index e010e7fe9..a2a28e979 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -881,7 +881,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.eurekaSDCheckInterval duration
     	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
@@ -1017,9 +1017,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 3cc86000e..191524df1 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -728,9 +728,9 @@ The shortlist of configuration flags is the following:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 16cd5e39b..0d23d56b6 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -167,7 +167,20 @@ func newManager(ctx context.Context) (*manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	nts, err := notifier.Init(alertURLGeneratorFn)
+
+	labels := make(map[string]string, 0)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -175,7 +188,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		groups:         make(map[uint64]*Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         map[string]string{},
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -189,16 +202,6 @@ func newManager(ctx context.Context) (*manager, error) {
 	}
 	manager.rr = rr
 
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
-		}
-		manager.labels[s[:n]] = s[n+1:]
-	}
 	return manager, nil
 }
 
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index c2889b94f..4aa661d24 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -70,7 +70,13 @@ type AlertTplData struct {
 	Expr   string
 }
 
-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+}
 
 // ExecTemplate executes the Alert template for given
 // map of annotations.
@@ -100,13 +106,15 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
 	for key, text := range annotations {
 		buf.Reset()
 		builder.Reset()
-		builder.Grow(len(tplHeader) + len(text))
-		builder.WriteString(tplHeader)
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -116,7 +124,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	return r, eg.Err()
 }
 
-func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error {
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
 	t := template.New("").Funcs(funcs).Option("missingkey=zero")
 	tpl, err := t.Parse(text)
 	if err != nil {
diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go
index d550de2fc..784751d02 100644
--- a/app/vmalert/notifier/init.go
+++ b/app/vmalert/notifier/init.go
@@ -46,13 +46,29 @@ func Reload() error {
 
 var staticNotifiersFn func() []Notifier
 
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   * configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
 //   * configuration via file. Supports live reloads and service discovery.
 // Init returns an error if both mods are used.
-func Init(gen AlertURLGenerator) (func() []Notifier, error) {
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	if externalLabels != nil || externalURL != "" {
+		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
+	}
+
+	externalURL = extURL
+	externalLabels = extLabels
+
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
 	}
diff --git a/app/vmauth/README.md b/app/vmauth/README.md
index ecb29d23f..c632e4a5f 100644
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -277,9 +277,9 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json
index de71f1494..b322a3db3 100644
--- a/dashboards/victoriametrics.json
+++ b/dashboards/victoriametrics.json
@@ -57,7 +57,7 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1644910726761,
+  "iteration": 1644936794057,
   "links": [
     {
       "icon": "doc",
@@ -2787,7 +2787,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2892,7 +2892,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2949,7 +2949,7 @@
           "datasource": {
             "uid": "$ds"
           },
-          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>50%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
+          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
           "fieldConfig": {
             "defaults": {
               "links": []
@@ -2985,7 +2985,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3001,7 +3001,17 @@
               "refId": "A"
             }
           ],
-          "thresholds": [],
+          "thresholds": [
+            {
+              "$$hashKey": "object:72",
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 0.1,
+              "yaxis": "left"
+            }
+          ],
           "timeRegions": [],
           "title": "Slow inserts",
           "tooltip": {
@@ -3079,7 +3089,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3186,7 +3196,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
@@ -3282,7 +3292,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3387,7 +3397,7 @@
             "alertThreshold": true
           },
           "percentage": false,
-          "pluginVersion": "8.3.2",
+          "pluginVersion": "8.3.5",
           "pointradius": 1,
           "points": false,
           "renderer": "flot",
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4d7777a45..3bf974a9d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -14,6 +14,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 ## tip
 
+
+* BUGFIX: vmalert: add support for `$externalLabels` and `$externalURL` template vars in the same way as Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2193).
 * BUGFIX: update default value for `-promscrape.fileSDCheckInterval`, so it matches default duration used by Prometheus for checking for updates in `file_sd_configs`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2187). Thanks to @corporate-gadfly for the fix.
 
 
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index eb5841453..5c2d18a03 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -590,9 +590,9 @@ Below is the output for `/path/to/vminsert -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
@@ -716,9 +716,9 @@ Below is the output for `/path/to/vmselect -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
@@ -819,9 +819,9 @@ Below is the output for `/path/to/vmstorage -help`:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
   -vminsertAddr string
diff --git a/docs/README.md b/docs/README.md
index 584b3e6d1..4473fba26 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -867,7 +867,7 @@ The [deduplication](#deduplication) isn't applied for the data exported in nativ
 
 ## How to import time series data
 
-Time series data can be imported via any supported ingestion protocol:
+Time series data can be imported into VictoriaMetrics via any supported ingestion protocol:
 
 * [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). See [these docs](#prometheus-setup) for details.
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
@@ -1778,7 +1778,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.eurekaSDCheckInterval duration
     	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
@@ -1900,9 +1900,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 7641205db..d98725443 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -871,7 +871,7 @@ The [deduplication](#deduplication) isn't applied for the data exported in nativ
 
 ## How to import time series data
 
-Time series data can be imported via any supported ingestion protocol:
+Time series data can be imported into VictoriaMetrics via any supported ingestion protocol:
 
 * [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). See [these docs](#prometheus-setup) for details.
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
@@ -1782,7 +1782,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.eurekaSDCheckInterval duration
     	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
@@ -1904,9 +1904,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/docs/guides/getting-started-with-vm-operator.md b/docs/guides/getting-started-with-vm-operator.md
index 365287de5..432cd961b 100644
--- a/docs/guides/getting-started-with-vm-operator.md
+++ b/docs/guides/getting-started-with-vm-operator.md
@@ -282,7 +282,7 @@ cat <<EOF | helm install my-grafana grafana/grafana -f -
     default:
       victoriametrics:
         gnetId: 11176
-        revision: 17
+        revision: 18
         datasource: victoriametrics
       vmagent:
         gnetId: 12683
diff --git a/docs/guides/k8s-monitoring-via-vm-cluster.md b/docs/guides/k8s-monitoring-via-vm-cluster.md
index 9fa0752e2..3ff58d63a 100644
--- a/docs/guides/k8s-monitoring-via-vm-cluster.md
+++ b/docs/guides/k8s-monitoring-via-vm-cluster.md
@@ -484,7 +484,7 @@ cat <<EOF | helm install my-grafana grafana/grafana -f -
     default:
       victoriametrics:
         gnetId: 11176
-        revision: 17
+        revision: 18
         datasource: victoriametrics
       vmagent:
         gnetId: 12683
diff --git a/docs/url-examples.md b/docs/url-examples.md
new file mode 100644
index 000000000..8bdb21f75
--- /dev/null
+++ b/docs/url-examples.md
@@ -0,0 +1,248 @@
+---
+sort: 20
+---
+
+# VictoriaMetrics API examples
+
+
+## /api/v1/admin/tsdb/delete_series
+
+**Deletes time series from VictoriaMetrics**
+ 
+Single:
+```bash
+curl 'http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total'
+```
+
+Cluster:
+```bash
+curl 'http://<vmselect>:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total'
+```
+
+Additional information:
+* [How to delete time series](https://docs.victoriametrics.com/#how-to-delete-time-series)
+
+
+## /api/v1/export/csv
+
+**Exports CSV data from VictoriaMetrics**
+ 
+Single:
+```bash
+curl 'http://<victoriametrics-addr>:8428/api/v1/export/csv?format=__name__,__value__,__timestamp__:unix_s&match=vm_http_request_errors_total' > filename.txt
+```
+ 
+Cluster:
+```bash
+curl -G 'http://<vmselect>:8481/select/0/prometheus/api/v1/export/csv?format=__name__,__value__,__timestamp__:unix_s&match=vm_http_request_errors_total' > filename.txt
+```
+
+Additional information: 
+* [How to export time series](https://docs.victoriametrics.com/#how-to-export-csv-data)
+* [URL Format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
+
+
+## /api/v1/export/native
+  
+**Exports data from VictoriaMetrics in native format**
+
+Single:
+```bash
+curl -G 'http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=vm_http_request_errors_total' > filename.txt
+```
+
+Cluster:
+```bash
+curl -G 'http://<vmselect>:8481/select/0/prometheus/api/v1/export/native?match=vm_http_request_errors_total' > filename.txt
+```
+
+More information:
+* [How to export data in native format](https://docs.victoriametrics.com/#how-to-export-data-in-native-format)
+
+
+## /api/v1/import
+
+**Imports data obtained via /api/v1/export**
+
+Single:
+```bash
+curl --data-binary "@import.txt" -X POST 'http://destination-victoriametrics:8428/api/v1/import'
+```
+
+Cluster:
+```bash
+curl --data-binary "@import.txt" -X POST 'http://<vminsert>:8480/insert/prometheus/api/v1/import'
+```
+
+Additional information:
+* [How to import time series data](https://docs.victoriametrics.com/#how-to-import-time-series-data)
+
+
+## /api/v1/import/csv 
+
+**Imports CSV data to VictoriaMetrics**
+ 
+Single:
+```bash
+curl --data-binary "@import.txt" -X POST 'http://localhost:8428/api/v1/import/prometheus'
+curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+```
+
+Cluster:
+```bash
+curl --data-binary "@import.txt" -X POST  'http://<vminsert>:8480/insert/0/prometheus/api/v1/import/csv'
+curl -d "GOOG,1.23,4.56,NYSE" 'http://<vminsert>:8480/insert/0/prometheus/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+```
+
+Additional information: 
+* [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
+* [How to import CSV data](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data)
+
+
+## /datadog/api/v1/series
+
+**Sends data from DataDog agent to VM**
+ 
+Single:
+```bash
+echo '
+{
+  "series": [
+    {
+      "host": "test.example.com",
+      "interval": 20,
+      "metric": "system.load.1",
+      "points": [[
+        0,
+        0.5
+      ]],
+      "tags": [
+        "environment:test"
+      ],
+      "type": "rate"
+    }
+  ]
+}
+' | curl -X POST --data-binary @- http://localhost:8428/datadog/api/v1/series
+```
+
+Cluster:
+```bash
+echo '
+{
+  "series": [
+    {
+      "host": "test.example.com",
+      "interval": 20,
+      "metric": "system.load.1",
+      "points": [[
+        0,
+        0.5
+      ]],
+      "tags": [
+        "environment:test"
+      ],
+      "type": "rate"
+    }
+  ]
+}
+' | curl -X POST --data-binary @- 'http://<vminsert>:8480/insert/0/datadog/api/v1/series'
+```
+
+Additional information:
+* [How to send data from datadog agent](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent)
+
+
+## /graphite/metrics/find
+
+**Searches Graphite metrics in VictoriaMetrics**
+
+Single:
+```bash
+curl -G 'http://localhost:8428/graphite/metrics/find?query=vm_http_request_errors_total'
+```
+ 
+Cluster:
+```bash
+curl -G 'http://<vmselect>:8481/select/0/graphite/metrics/find?query=vm_http_request_errors_total'
+```
+ 
+Additional information:
+* [Metrics find](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find)
+* [How to send data from graphite compatible agents such as statsd](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
+* [URL Format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
+
+
+## /influx/write
+
+**Writes data with InfluxDB line protocol to VictoriaMetrics**
+
+Single:
+```bash
+curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'http://localhost:8428/write'
+```
+ 
+Cluster:
+```bash
+curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'http://<vminsert>:8480/insert/0/influx/write'
+```
+ 
+Additional information:
+* [How to send data from influxdb compatible agents such as telegraf](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+
+
+## TCP and UDP
+
+**How to send data from OpenTSDB-compatible agents to VictoriaMetrics**
+
+Turned off by default. Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command-line flag.
+*If run from docker, '-opentsdbListenAddr' port should be exposed*
+
+Single:
+```bash
+echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N localhost 4242
+```
+ 
+Cluster:
+```bash
+echo "put foo.bar.baz `date +%s` 123  tag1=value1 tag2=value2 VictoriaMetrics_AccountID=0" | nc -N http://<vminsert> 4242
+```
+ 
+Enable HTTP server for OpenTSDB /api/put requests by setting `-opentsdbHTTPListenAddr` command-line flag.
+ 
+Single:
+```bash
+curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
+```
+ 
+Cluster:
+```bash
+curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]'
+ 'http://<vminsert>:8480/insert/42/opentsdb/api/put'
+```
+ 
+Additional information:
+* [Api http put](http://opentsdb.net/docs/build/html/api_http/put.html)
+* [How to send data from opentsdb compatible agents](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents)
+
+
+**How to write data with Graphite plaintext protocol to VictoriaMetrics**
+
+Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command-line flag.
+ 
+Single:
+```bash
+echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" |
+ nc -N localhost 2003
+```
+ 
+Cluster:
+```bash
+echo "foo.bar.baz;tag1=value1;tag2=value2;VictoriaMetrics_AccountID=42 123 `date +%s`" | nc -N http://<vminsert> 2003
+```
+
+Additional information:
+
+`VictoriaMetrics_AccountID=42` - tag that indicated tenant ID.
+* [Request handler](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/a3eafd2e7fc75776dfc19d3c68c85589454d9dce/app/vminsert/opentsdb/request_handler.go#L47)
+* [How to send data from graphite compatible agents such as statsd](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
diff --git a/docs/vmagent.md b/docs/vmagent.md
index e1bab0e7d..c03341a20 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -885,7 +885,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.eurekaSDCheckInterval duration
     	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
@@ -1021,9 +1021,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/docs/vmalert.md b/docs/vmalert.md
index 060d92009..1cf438a2a 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -732,9 +732,9 @@ The shortlist of configuration flags is the following:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
diff --git a/docs/vmauth.md b/docs/vmauth.md
index 4e087fed4..dc6329670 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -281,9 +281,9 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```


Test patch:
diff --git a/app/vmalert/config/testdata/rules2-good.rules b/app/vmalert/config/testdata/rules2-good.rules
index c307375ac..1f431b32f 100644
--- a/app/vmalert/config/testdata/rules2-good.rules
+++ b/app/vmalert/config/testdata/rules2-good.rules
@@ -25,6 +25,7 @@ groups:
           dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}'
         annotations:
           description: Job {{ $labels.job }} is up!
+          external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }}
           summary: All instances up {{ range query "up" }}
             {{ . | label "instance" }}
             {{ end }}
diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go
index 769116979..f8e0c77ee 100644
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@@ -1,12 +1,24 @@
 package notifier
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string, 0)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
 	testCases := []struct {
 		name        string
 		alert       *Alert
@@ -74,6 +86,26 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"desc":    "bar 1;garply 2;",
 			},
 		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
 	}
 
 	qFn := func(q string) ([]datasource.Metric, error) {