Repository: VictoriaMetrics/VictoriaMetrics
Pull request #965 (2020-12-14 17:45:33)

adds new Array Flags
ArrayDuration and ArrayBool flags,
makes sendTimeout and tlsInsecure configurable per remoteWriteURL


Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 669de3f20..4662926ee 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -20,11 +20,11 @@ jobs:
         id: go
       - name: Dependencies
         env:
-          GO111MODULE: on
+          GO111MODULE: off
         run: |
-          go get -u golang.org/x/lint/golint
-          go get -u github.com/kisielk/errcheck
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
+          go install golang.org/x/lint/golint
+          go install github.com/kisielk/errcheck
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.29.0
       - name: Code checkout
         uses: actions/checkout@master
       - name: Build
diff --git a/Makefile b/Makefile
index 3257e7777..b20c241fa 100644
--- a/Makefile
+++ b/Makefile
@@ -80,7 +80,7 @@ lint: install-golint
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
+	which golint || GO111MODULE=off go install golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
@@ -94,7 +94,7 @@ errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
 
 install-errcheck:
-	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
+	which errcheck || GO111MODULE=off go install github.com/kisielk/errcheck
 
 check-all: fmt vet lint errcheck golangci-lint
 
@@ -140,14 +140,14 @@ quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
+	which qtc || GO111MODULE=off go install github.com/valyala/quicktemplate/qtc
 
 
 golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.29.0
 
 docs-sync:
 	cp app/vmagent/README.md docs/vmagent.md
diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go
index a169f774b..2ffdfab12 100644
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/base64"
-	"flag"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -21,11 +20,11 @@ import (
 )
 
 var (
-	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to -remoteWrite.url")
 	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. "+
 		"Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
@@ -108,7 +107,7 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		fq:             fq,
 		hc: &http.Client{
 			Transport: tr,
-			Timeout:   *sendTimeout,
+			Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
 		},
 		stopCh: make(chan struct{}),
 	}
@@ -140,7 +139,7 @@ func getTLSConfig(argIdx int) (*tls.Config, error) {
 		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
 		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
 		ServerName:         tlsServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: *tlsInsecureSkipVerify,
+		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
 	if c.CAFile == "" && c.CertFile == "" && c.KeyFile == "" && c.ServerName == "" && !c.InsecureSkipVerify {
 		return nil, nil
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 4c93979c3..773873039 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -21,7 +21,6 @@ may fail;
 * by default, rules execution is sequential within one group, but persisting of execution results to remote
 storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
 recording rule is reused in next one;
-* there is no `query` function support in templates yet;
 * `vmalert` has no UI, just an API for getting groups and rules statuses.
 
 ### QuickStart
diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go
index 3b607eb4c..592d35226 100644
--- a/app/vmalert/alerting.go
+++ b/app/vmalert/alerting.go
@@ -137,6 +137,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 		}
 	}
 
+	qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query) }
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
@@ -158,14 +159,14 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 				a.Value = m.Value
 				// and re-exec template since Value can be used
 				// in templates
-				err = ar.template(a)
+				err = ar.template(a, qFn)
 				if err != nil {
 					return nil, err
 				}
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ar.lastExecTime)
+		a, err := ar.newAlert(m, ar.lastExecTime, qFn)
 		if err != nil {
 			ar.lastExecError = err
 			return nil, fmt.Errorf("failed to create alert: %w", err)
@@ -245,7 +246,7 @@ func hash(m datasource.Metric) uint64 {
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifier.Alert, error) {
+func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
 	a := &notifier.Alert{
 		GroupID: ar.GroupID,
 		Name:    ar.Name,
@@ -264,16 +265,16 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifie
 		}
 		a.Labels[l.Name] = l.Value
 	}
-	return a, ar.template(a)
+	return a, ar.template(a, qFn)
 }
 
-func (ar *AlertingRule) template(a *notifier.Alert) error {
+func (ar *AlertingRule) template(a *notifier.Alert, qFn notifier.QueryFn) error {
 	var err error
-	a.Labels, err = a.ExecTemplate(a.Labels)
+	a.Labels, err = a.ExecTemplate(qFn, a.Labels)
 	if err != nil {
 		return err
 	}
-	a.Annotations, err = a.ExecTemplate(ar.Annotations)
+	a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
 	return err
 }
 
@@ -393,6 +394,8 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		return fmt.Errorf("querier is nil")
 	}
 
+	qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query) }
+
 	// account for external labels in filter
 	var labelsFilter string
 	for k, v := range labels {
@@ -421,7 +424,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 			m.Labels = append(m.Labels, l)
 		}
 
-		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0))
+		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
diff --git a/app/vmalert/datasource/datasource.go b/app/vmalert/datasource/datasource.go
index de0265d24..5e6aa32d3 100644
--- a/app/vmalert/datasource/datasource.go
+++ b/app/vmalert/datasource/datasource.go
@@ -34,6 +34,17 @@ func (m *Metric) AddLabel(key, value string) {
 	m.Labels = append(m.Labels, Label{Name: key, Value: value})
 }
 
+// Label returns the given label value.
+// If label is missing empty string will be returned
+func (m *Metric) Label(key string) string {
+	for _, l := range m.Labels {
+		if l.Name == key {
+			return l.Value
+		}
+	}
+	return ""
+}
+
 // Label represents metric's label
 type Label struct {
 	Name  string
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index d90a3f31c..ea6922d86 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -206,7 +206,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(m)
+		templated, err := alert.ExecTemplate(nil, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index 04d4d0a71..e71b549ea 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -62,21 +62,23 @@ const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Ex
 
 // ExecTemplate executes the Alert template for give
 // map of annotations.
-func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
+// Every alert could have a different datasource, so function
+// requires a queryFunction as an argument.
+func (a *Alert) ExecTemplate(q QueryFn, annotations map[string]string) (map[string]string, error) {
 	tplData := alertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
-	return templateAnnotations(annotations, tplHeader, tplData)
+	return templateAnnotations(annotations, tplData, funcsWithQuery(q))
 }
 
 // ValidateTemplates validate annotations for possible template error, uses empty data for template population
 func ValidateTemplates(annotations map[string]string) error {
-	_, err := templateAnnotations(annotations, tplHeader, alertTplData{
+	_, err := templateAnnotations(annotations, alertTplData{
 		Labels: map[string]string{},
 		Value:  0,
-	})
+	}, tmplFunc)
 	return err
 }
 
-func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
+func templateAnnotations(annotations map[string]string, data alertTplData, funcs template.FuncMap) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
@@ -85,10 +87,10 @@ func templateAnnotations(annotations map[string]string, header string, data aler
 		r[key] = text
 		buf.Reset()
 		builder.Reset()
-		builder.Grow(len(header) + len(text))
-		builder.WriteString(header)
+		builder.Grow(len(tplHeader) + len(text))
+		builder.WriteString(tplHeader)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
 		}
@@ -97,8 +99,9 @@ func templateAnnotations(annotations map[string]string, header string, data aler
 	return r, eg.Err()
 }
 
-func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
-	tpl, err := template.New("").Funcs(tmplFunc).Option("missingkey=zero").Parse(text)
+func templateAnnotation(dst io.Writer, text string, data alertTplData, funcs template.FuncMap) error {
+	t := template.New("").Funcs(funcs).Option("missingkey=zero")
+	tpl, err := t.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation: %w", err)
 	}
diff --git a/app/vmalert/notifier/template_func.go b/app/vmalert/notifier/template_func.go
index abfd8fc91..e643d5880 100644
--- a/app/vmalert/notifier/template_func.go
+++ b/app/vmalert/notifier/template_func.go
@@ -14,21 +14,40 @@
 package notifier
 
 import (
+	"errors"
 	"fmt"
-	html_template "html/template"
 	"math"
 	"net/url"
 	"regexp"
 	"strings"
-	text_template "text/template"
 	"time"
+
+	htmlTpl "html/template"
+	textTpl "text/template"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
-var tmplFunc text_template.FuncMap
+// QueryFn is used to wrap a call to datasource into simple-to-use function
+// for templating functions.
+type QueryFn func(query string) ([]datasource.Metric, error)
+
+func funcsWithQuery(query QueryFn) textTpl.FuncMap {
+	fm := make(textTpl.FuncMap)
+	for k, fn := range tmplFunc {
+		fm[k] = fn
+	}
+	fm["query"] = func(q string) ([]datasource.Metric, error) {
+		return query(q)
+	}
+	return fm
+}
+
+var tmplFunc textTpl.FuncMap
 
-// InitTemplateFunc returns template helper functions
+// InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
-	tmplFunc = text_template.FuncMap{
+	tmplFunc = textTpl.FuncMap{
 		"args": func(args ...interface{}) map[string]interface{} {
 			result := make(map[string]interface{})
 			for i, a := range args {
@@ -40,8 +59,8 @@ func InitTemplateFunc(externalURL *url.URL) {
 			re := regexp.MustCompile(pattern)
 			return re.ReplaceAllString(text, repl)
 		},
-		"safeHtml": func(text string) html_template.HTML {
-			return html_template.HTML(text)
+		"safeHtml": func(text string) htmlTpl.HTML {
+			return htmlTpl.HTML(text)
 		},
 		"match":   regexp.MatchString,
 		"title":   strings.Title,
@@ -151,6 +170,24 @@ func InitTemplateFunc(externalURL *url.URL) {
 		"quotesEscape": func(q string) string {
 			return strings.Replace(q, `"`, `\"`, -1)
 		},
+		// query function supposed to be substituted at funcsWithQuery().
+		// it is present here only for validation purposes, when there is no
+		// provided datasource.
+		"query": func(q string) ([]datasource.Metric, error) {
+			return nil, nil
+		},
+		"first": func(metrics []datasource.Metric) (datasource.Metric, error) {
+			if len(metrics) > 0 {
+				return metrics[0], nil
+			}
+			return datasource.Metric{}, errors.New("first() called on vector with no elements")
+		},
+		"label": func(label string, m datasource.Metric) string {
+			return m.Label(label)
+		},
+		"value": func(m datasource.Metric) float64 {
+			return m.Value
+		},
 	}
 }
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5a9a46dc8..c24f4458e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -3,6 +3,7 @@
 # tip
 
 * FEATURE: automatically reset response cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to VictoriaMetrics. This makes unnecessary disabling response cache during data backfilling or resetting it after backfilling is complete as described [in these docs](https://victoriametrics.github.io/#backfilling). This feature applies only to single-node VictoriaMetrics. It doesn't apply to cluster version of VictoriaMetrics because `vminsert` nodes don't know about `vmselect` nodes where the response cache must be reset.
+* FEATURE: vmalert: add `query`, `first` and `value` functions to alert templates. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/539
 * FEATURE: vmagent: return user-friendly HTML page when requesting `/targets` page from web browser. The page is returned in the old plaintext format when requesting via curl or similar tool.
 * FEATURE: allow multiple whitespace chars between measurements, fields and timestamp when parsing InfluxDB line protocol.
   Though [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) denies multiple whitespace chars between these entities,
diff --git a/lib/flagutil/array.go b/lib/flagutil/array.go
index d4454795f..51f7d2f5b 100644
--- a/lib/flagutil/array.go
+++ b/lib/flagutil/array.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // NewArray returns new Array with the given name and description.
@@ -16,6 +17,24 @@ func NewArray(name, description string) *Array {
 	return &a
 }
 
+// NewArrayDuration returns new ArrayDuration with the given name and description.
+func NewArrayDuration(name, description string) *ArrayDuration {
+	description += "\nSupports `array` of values separated by comma" +
+		" or specified via multiple flags."
+	var a ArrayDuration
+	flag.Var(&a, name, description)
+	return &a
+}
+
+// NewArrayBool returns new ArrayBool with the given name and description.
+func NewArrayBool(name, description string) *ArrayBool {
+	description += "\nSupports `array` of values separated by comma" +
+		" or specified via multiple flags."
+	var a ArrayBool
+	flag.Var(&a, name, description)
+	return &a
+}
+
 // Array is a flag that holds an array of values.
 //
 // It may be set either by specifying multiple flags with the given name
@@ -124,3 +143,83 @@ func (a *Array) GetOptionalArg(argIdx int) string {
 	}
 	return x[argIdx]
 }
+
+// ArrayBool is a flag that holds an array of booleans values.
+// have the same api as Array.
+type ArrayBool []bool
+
+// IsBoolFlag  implements flag.IsBoolFlag interface
+func (a *ArrayBool) IsBoolFlag() bool { return true }
+
+// String implements flag.Value interface
+func (a *ArrayBool) String() string {
+	formattedBools := make([]string, len(*a))
+	for i, v := range *a {
+		formattedBools[i] = strconv.FormatBool(v)
+	}
+	return strings.Join(formattedBools, ",")
+}
+
+// Set implements flag.Value interface
+func (a *ArrayBool) Set(value string) error {
+	values := parseArrayValues(value)
+	for _, v := range values {
+		b, err := strconv.ParseBool(v)
+		if err != nil {
+			return err
+		}
+		*a = append(*a, b)
+	}
+	return nil
+}
+
+// GetOptionalArg returns optional arg under the given argIdx.
+func (a *ArrayBool) GetOptionalArg(argIdx int) bool {
+	x := *a
+	if argIdx >= len(x) {
+		if len(x) == 1 {
+			return x[0]
+		}
+		return false
+	}
+	return x[argIdx]
+}
+
+// ArrayDuration is a flag that holds an array of time.Duration values.
+// have the same api as Array.
+type ArrayDuration []time.Duration
+
+// String implements flag.Value interface
+func (a *ArrayDuration) String() string {
+	formattedBools := make([]string, len(*a))
+	for i, v := range *a {
+		formattedBools[i] = v.String()
+	}
+	return strings.Join(formattedBools, ",")
+}
+
+// Set implements flag.Value interface
+func (a *ArrayDuration) Set(value string) error {
+	values := parseArrayValues(value)
+	for _, v := range values {
+		b, err := time.ParseDuration(v)
+		if err != nil {
+			return err
+		}
+		*a = append(*a, b)
+	}
+	return nil
+}
+
+// GetOptionalArgOrDefault returns optional arg under the given argIdx,
+// or default value, if argIdx not found.
+func (a *ArrayDuration) GetOptionalArgOrDefault(argIdx int, defaultValue time.Duration) time.Duration {
+	x := *a
+	if argIdx >= len(x) {
+		if len(x) == 1 {
+			return x[0]
+		}
+		return defaultValue
+	}
+	return x[argIdx]
+}
diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl
index f1fa91f9b..fabd7b103 100644
--- a/lib/promscrape/targets_response.qtpl
+++ b/lib/promscrape/targets_response.qtpl
@@ -1,12 +1,12 @@
 {% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 %}
 
-{% stripspace %}
+{% collapsespace %}
 
 {% func TargetsResponsePlain (jts []jobTargetsStatuses,  showOriginLabels bool) -%}
 
 {% for _, js := range jts %}
-job={%q= js.job %}{% space %} ({%d js.upCount %}/{%d js.targetsTotal %} {% space %} up)
+job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)
 {% newline %}
 {% for _, ts := range js.targetsStatus %}
     {% code
@@ -14,12 +14,12 @@ job={%q= js.job %}{% space %} ({%d js.upCount %}/{%d js.targetsTotal %} {% space
      ol := promLabelsString(ts.originalLabels)
     %}
 {%s= "\t" %}state={% if ts.up %}up{% else %}down{% endif %},
-    {% space %} endpoint={%s= ts.endpoint %},
-    {% space %} labels={%s= labels %}
-    {% if showOriginLabels %},{% space %} originalLabels={%s= ol %}{% endif %},
-    {% space %} last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s {% space %} ago,
-    {% space %} scrape_duration={%f.3 float64(ts.scrapeDuration.Seconds()) %}s,
-    {% space %} error={%q= ts.error %}
+    endpoint={%s= ts.endpoint %},
+    labels={%s= labels %}
+    {% if showOriginLabels %}, originalLabels={%s= ol %}{% endif %},
+    last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago,
+    scrape_duration={%f.3 ts.scrapeDuration.Seconds() %}s,
+    error={%q= ts.error %}
     {% newline %}
 {% endfor %}
 {% endfor %}
@@ -29,76 +29,68 @@ job={%q= js.job %}{% space %} ({%d js.upCount %}/{%d js.targetsTotal %} {% space
 
 {% func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) %}
 <!DOCTYPE html>
-<style>
-    .border{
-        border-collapse: collapse;
-        border: 1px solid black;
-    }
-    .table-row:hover{
-        background-color: #f5f5f5;
-    }
-</style>
 <html lang="en">
-    <head>
-        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-        <title>VictoriaMetrics Database</title>
-    </head>
-    <body>
-    <h1>Targets</h1>
-    <div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons">
-      <label class="btn">
-        <input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}';" {% if !onlyUnhealthy %}checked {% endif %}> All
-      </label>
-      <label class="btn">
-        <input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}?show_only_unhealthy=true';" {% if onlyUnhealthy %}checked {% endif %}> Unhealthy
-      </label>
-      <br />
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-giJF6kkoqNQ00vy+HMDP7azOuL0xtbfIcaT9wjKHr8RbDVddVHyTfAAsrekwKmP1" crossorigin="anonymous">
+    <title>Scrape targets</title>
+</head>
+<body class="m-3">
+  <h1>Scrape targets</h1>
+  <div>
+    <button type="button" class="btn {% if !onlyUnhealthy %}btn-primary{% else %}btn-secondary{% endif %}"
+        {% if onlyUnhealthy %}onclick="location.href='{%s redirectPath %}'"{% endif %}>
+        All
+    </button>
+    <button type="button" class="btn {% if onlyUnhealthy %}btn-primary{% else %}btn-secondary{% endif %}"
+        {% if !onlyUnhealthy %}onclick="location.href='{%s= redirectPath %}?show_only_unhealthy=true'"{% endif %}>
+        Unhealthy
+    </button>
   </div>
   {% for _,js :=range jts %}
-  		    <div class="table-container">
-                <h2 class="job_header danger">
-                  <a id="job-{%q= js.job %}" >{%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)</a>
-                </h2>
-                <table class="table-bordered table-hover border">
-                  <thead class="job_details border">
-                    <tr class="table-row border">
-                      <th class="border">Endpoint</th>
-                      <th class="border">State</th>
-                      <th class="border">Labels</th>
-                      <th class="border">Last Scrape</th>
-                      <th class="border">Scrape Duration</th>
-                      <th class="border">Error</th>
-                    </tr>
-                  </thead>
-                  <tbody>
-                  {% for _, ts := range js.targetsStatus %}
-                    {% if onlyUnhealthy && ts.up %} {% continue %} {% endif %}
-                    <tr class="table-row border">
-                      <td class="endpoint border">
-                        <a href="{%s= ts.endpoint %}">{%s= ts.endpoint %}</a><br>
-                      </td>
-                      <td class="state border">
-                        <span class="state_indicator">{% if ts.up %}UP{% else %}DOWN{% endif %}</span>
-                      </td>
-                      <td class="labels border", title="Original {% space %} labels: {% space %} {%= formatLabel(ts.originalLabels) %}">
-                           {%= formatLabel(ts.labels) %}
-                      </td>
-                      <td class="last-scrape border">{%s ts.lastScrapeTime.String() %} {% space %} ago</td>
-                      <td class="scrape-duration border">{%s ts.scrapeDuration.String() %}</td>
-                      <td class="errors border"><span class="alert alert-danger state_indicator">{%s= ts.error %}</span></td>
-                    </tr>
-                  {% endfor %}
-                  </tbody>
-                </table>
-              </div>
+    {% if onlyUnhealthy && js.upCount == js.targetsTotal %}{% continue %}{% endif %}
+    <div>
+      <h4>
+        <a>{%s js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)</a>
+      </h4>
+      <table class="table table-striped table-hover table-bordered table-sm">
+        <thead>
+          <tr>
+            <th scope="col">Endpoint</th>
+            <th scope="col">State</th>
+            <th scope="col">Labels</th>
+            <th scope="col">Last Scrape</th>
+            <th scope="col">Scrape Duration</th>
+            <th scope="col">Error</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for _, ts := range js.targetsStatus %}
+            {% if onlyUnhealthy && ts.up %}{% continue %}{% endif %}
+            <tr {% if !ts.up %}class="alert alert-danger" role="alert"{% endif %}>
+              <td><a href="{%s ts.endpoint %}">{%s ts.endpoint %}</a><br></td>
+              <td>{% if ts.up %}UP{% else %}DOWN{% endif %}</td>
+              <td title="Original labels: {%= formatLabel(ts.originalLabels) %}">
+                {%= formatLabel(ts.labels) %}
+              </td>
+              <td>{%f.3 ts.lastScrapeTime.Seconds() %}s ago</td>
+              <td>{%f.3 ts.scrapeDuration.Seconds() %}s</td>
+              <td>{%s ts.error %}</td>
+            </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
   {% endfor %}
-    </body>
+</body>
 </html>
 {% endfunc %}
 
 {% func formatLabel(labels []prompbmarshal.Label) %}
 {% for _, label := range labels %}
- {% space %} {%s label.Name %}={%q label.Value %} {% space %}
+    {%s label.Name %}={%q label.Value %} {% space %}
 {% endfor %}
 {% endfunc %}
-{% endstripspace %}
+
+{% endcollapsespace %}
diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go
index b0472bddb..0c9d9699a 100644
--- a/lib/promscrape/targets_response.qtpl.go
+++ b/lib/promscrape/targets_response.qtpl.go
@@ -25,13 +25,11 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu
 //line lib/promscrape/targets_response.qtpl:8
 	for _, js := range jts {
 //line lib/promscrape/targets_response.qtpl:8
-		qw422016.N().S(`job=`)
+		qw422016.N().S(` job=`)
 //line lib/promscrape/targets_response.qtpl:9
 		qw422016.N().Q(js.job)
 //line lib/promscrape/targets_response.qtpl:9
-		qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:9
-		qw422016.N().S(`(`)
+		qw422016.N().S(` (`)
 //line lib/promscrape/targets_response.qtpl:9
 		qw422016.N().D(js.upCount)
 //line lib/promscrape/targets_response.qtpl:9
@@ -39,18 +37,22 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu
 //line lib/promscrape/targets_response.qtpl:9
 		qw422016.N().D(js.targetsTotal)
 //line lib/promscrape/targets_response.qtpl:9
-		qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:9
-		qw422016.N().S(`up)`)
+		qw422016.N().S(` up) `)
 //line lib/promscrape/targets_response.qtpl:10
 		qw422016.N().S(`
 `)
+//line lib/promscrape/targets_response.qtpl:10
+		qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:11
 		for _, ts := range js.targetsStatus {
+//line lib/promscrape/targets_response.qtpl:11
+			qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:13
 			labels := promLabelsString(ts.labels)
 			ol := promLabelsString(ts.originalLabels)
 
+//line lib/promscrape/targets_response.qtpl:15
+			qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:16
 			qw422016.N().S("\t")
 //line lib/promscrape/targets_response.qtpl:16
@@ -66,71 +68,55 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu
 //line lib/promscrape/targets_response.qtpl:16
 			}
 //line lib/promscrape/targets_response.qtpl:16
-			qw422016.N().S(`,`)
-//line lib/promscrape/targets_response.qtpl:17
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:17
-			qw422016.N().S(`endpoint=`)
+			qw422016.N().S(`, endpoint=`)
 //line lib/promscrape/targets_response.qtpl:17
 			qw422016.N().S(ts.endpoint)
 //line lib/promscrape/targets_response.qtpl:17
-			qw422016.N().S(`,`)
-//line lib/promscrape/targets_response.qtpl:18
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:18
-			qw422016.N().S(`labels=`)
+			qw422016.N().S(`, labels=`)
 //line lib/promscrape/targets_response.qtpl:18
 			qw422016.N().S(labels)
+//line lib/promscrape/targets_response.qtpl:18
+			qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:19
 			if showOriginLabels {
 //line lib/promscrape/targets_response.qtpl:19
-				qw422016.N().S(`,`)
-//line lib/promscrape/targets_response.qtpl:19
-				qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:19
-				qw422016.N().S(`originalLabels=`)
+				qw422016.N().S(`, originalLabels=`)
 //line lib/promscrape/targets_response.qtpl:19
 				qw422016.N().S(ol)
 //line lib/promscrape/targets_response.qtpl:19
 			}
 //line lib/promscrape/targets_response.qtpl:19
-			qw422016.N().S(`,`)
-//line lib/promscrape/targets_response.qtpl:20
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:20
-			qw422016.N().S(`last_scrape=`)
+			qw422016.N().S(`, last_scrape=`)
 //line lib/promscrape/targets_response.qtpl:20
 			qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
 //line lib/promscrape/targets_response.qtpl:20
-			qw422016.N().S(`s`)
-//line lib/promscrape/targets_response.qtpl:20
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:20
-			qw422016.N().S(`ago,`)
+			qw422016.N().S(`s ago, scrape_duration=`)
 //line lib/promscrape/targets_response.qtpl:21
-			qw422016.N().S(` `)
+			qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3)
 //line lib/promscrape/targets_response.qtpl:21
-			qw422016.N().S(`scrape_duration=`)
-//line lib/promscrape/targets_response.qtpl:21
-			qw422016.N().FPrec(float64(ts.scrapeDuration.Seconds()), 3)
-//line lib/promscrape/targets_response.qtpl:21
-			qw422016.N().S(`s,`)
-//line lib/promscrape/targets_response.qtpl:22
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:22
-			qw422016.N().S(`error=`)
+			qw422016.N().S(`s, error=`)
 //line lib/promscrape/targets_response.qtpl:22
 			qw422016.N().Q(ts.error)
+//line lib/promscrape/targets_response.qtpl:22
+			qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:23
 			qw422016.N().S(`
 `)
+//line lib/promscrape/targets_response.qtpl:23
+			qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:24
 		}
+//line lib/promscrape/targets_response.qtpl:24
+		qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:25
 	}
+//line lib/promscrape/targets_response.qtpl:25
+	qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:26
 	qw422016.N().S(`
 `)
+//line lib/promscrape/targets_response.qtpl:26
+	qw422016.N().S(` `)
 //line lib/promscrape/targets_response.qtpl:28
 }
 
@@ -163,187 +149,225 @@ func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) strin
 //line lib/promscrape/targets_response.qtpl:30
 func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
 //line lib/promscrape/targets_response.qtpl:30
-	qw422016.N().S(`<!DOCTYPE html><style>.border{border-collapse: collapse;border: 1px solid black;}.table-row:hover{background-color: #f5f5f5;}</style><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>VictoriaMetrics Database</title></head><body><h1>Targets</h1><div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons"><label class="btn"><input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='`)
-//line lib/promscrape/targets_response.qtpl:50
-	qw422016.N().S(redirectPath)
-//line lib/promscrape/targets_response.qtpl:50
-	qw422016.N().S(`';"`)
-//line lib/promscrape/targets_response.qtpl:50
+	qw422016.N().S(` <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-giJF6kkoqNQ00vy+HMDP7azOuL0xtbfIcaT9wjKHr8RbDVddVHyTfAAsrekwKmP1" crossorigin="anonymous"> <title>Scrape targets</title> </head> <body class="m-3"> <h1>Scrape targets</h1> <div> <button type="button" class="btn `)
+//line lib/promscrape/targets_response.qtpl:42
 	if !onlyUnhealthy {
-//line lib/promscrape/targets_response.qtpl:50
-		qw422016.N().S(`checked`)
-//line lib/promscrape/targets_response.qtpl:50
+//line lib/promscrape/targets_response.qtpl:42
+		qw422016.N().S(`btn-primary`)
+//line lib/promscrape/targets_response.qtpl:42
+	} else {
+//line lib/promscrape/targets_response.qtpl:42
+		qw422016.N().S(`btn-secondary`)
+//line lib/promscrape/targets_response.qtpl:42
+	}
+//line lib/promscrape/targets_response.qtpl:42
+	qw422016.N().S(`" `)
+//line lib/promscrape/targets_response.qtpl:43
+	if onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:43
+		qw422016.N().S(`onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:43
+		qw422016.E().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:43
+		qw422016.N().S(`'"`)
+//line lib/promscrape/targets_response.qtpl:43
 	}
-//line lib/promscrape/targets_response.qtpl:50
-	qw422016.N().S(`> All</label><label class="btn"><input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='`)
-//line lib/promscrape/targets_response.qtpl:53
-	qw422016.N().S(redirectPath)
-//line lib/promscrape/targets_response.qtpl:53
-	qw422016.N().S(`?show_only_unhealthy=true';"`)
-//line lib/promscrape/targets_response.qtpl:53
+//line lib/promscrape/targets_response.qtpl:43
+	qw422016.N().S(`> All </button> <button type="button" class="btn `)
+//line lib/promscrape/targets_response.qtpl:46
 	if onlyUnhealthy {
-//line lib/promscrape/targets_response.qtpl:53
-		qw422016.N().S(`checked`)
-//line lib/promscrape/targets_response.qtpl:53
+//line lib/promscrape/targets_response.qtpl:46
+		qw422016.N().S(`btn-primary`)
+//line lib/promscrape/targets_response.qtpl:46
+	} else {
+//line lib/promscrape/targets_response.qtpl:46
+		qw422016.N().S(`btn-secondary`)
+//line lib/promscrape/targets_response.qtpl:46
 	}
-//line lib/promscrape/targets_response.qtpl:53
-	qw422016.N().S(`> Unhealthy</label><br /></div>`)
-//line lib/promscrape/targets_response.qtpl:57
+//line lib/promscrape/targets_response.qtpl:46
+	qw422016.N().S(`" `)
+//line lib/promscrape/targets_response.qtpl:47
+	if !onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:47
+		qw422016.N().S(`onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:47
+		qw422016.N().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:47
+		qw422016.N().S(`?show_only_unhealthy=true'"`)
+//line lib/promscrape/targets_response.qtpl:47
+	}
+//line lib/promscrape/targets_response.qtpl:47
+	qw422016.N().S(`> Unhealthy </button> </div> `)
+//line lib/promscrape/targets_response.qtpl:51
 	for _, js := range jts {
-//line lib/promscrape/targets_response.qtpl:57
-		qw422016.N().S(`<div class="table-container"><h2 class="job_header danger"><a id="job-`)
-//line lib/promscrape/targets_response.qtpl:60
-		qw422016.N().Q(js.job)
-//line lib/promscrape/targets_response.qtpl:60
-		qw422016.N().S(`" >`)
-//line lib/promscrape/targets_response.qtpl:60
-		qw422016.N().Q(js.job)
-//line lib/promscrape/targets_response.qtpl:60
-		qw422016.N().S(`(`)
-//line lib/promscrape/targets_response.qtpl:60
+//line lib/promscrape/targets_response.qtpl:51
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:52
+		if onlyUnhealthy && js.upCount == js.targetsTotal {
+//line lib/promscrape/targets_response.qtpl:52
+			continue
+//line lib/promscrape/targets_response.qtpl:52
+		}
+//line lib/promscrape/targets_response.qtpl:52
+		qw422016.N().S(` <div> <h4> <a>`)
+//line lib/promscrape/targets_response.qtpl:55
+		qw422016.E().S(js.job)
+//line lib/promscrape/targets_response.qtpl:55
+		qw422016.N().S(` (`)
+//line lib/promscrape/targets_response.qtpl:55
 		qw422016.N().D(js.upCount)
-//line lib/promscrape/targets_response.qtpl:60
+//line lib/promscrape/targets_response.qtpl:55
 		qw422016.N().S(`/`)
-//line lib/promscrape/targets_response.qtpl:60
+//line lib/promscrape/targets_response.qtpl:55
 		qw422016.N().D(js.targetsTotal)
-//line lib/promscrape/targets_response.qtpl:60
-		qw422016.N().S(`up)</a></h2><table class="table-bordered table-hover border"><thead class="job_details border"><tr class="table-row border"><th class="border">Endpoint</th><th class="border">State</th><th class="border">Labels</th><th class="border">Last Scrape</th><th class="border">Scrape Duration</th><th class="border">Error</th></tr></thead><tbody>`)
-//line lib/promscrape/targets_response.qtpl:74
+//line lib/promscrape/targets_response.qtpl:55
+		qw422016.N().S(` up)</a> </h4> <table class="table table-striped table-hover table-bordered table-sm"> <thead> <tr> <th scope="col">Endpoint</th> <th scope="col">State</th> <th scope="col">Labels</th> <th scope="col">Last Scrape</th> <th scope="col">Scrape Duration</th> <th scope="col">Error</th> </tr> </thead> <tbody> `)
+//line lib/promscrape/targets_response.qtpl:69
 		for _, ts := range js.targetsStatus {
-//line lib/promscrape/targets_response.qtpl:75
+//line lib/promscrape/targets_response.qtpl:69
+			qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:70
 			if onlyUnhealthy && ts.up {
-//line lib/promscrape/targets_response.qtpl:75
+//line lib/promscrape/targets_response.qtpl:70
 				continue
-//line lib/promscrape/targets_response.qtpl:75
+//line lib/promscrape/targets_response.qtpl:70
 			}
-//line lib/promscrape/targets_response.qtpl:75
-			qw422016.N().S(`<tr class="table-row border"><td class="endpoint border"><a href="`)
-//line lib/promscrape/targets_response.qtpl:78
-			qw422016.N().S(ts.endpoint)
-//line lib/promscrape/targets_response.qtpl:78
+//line lib/promscrape/targets_response.qtpl:70
+			qw422016.N().S(` <tr `)
+//line lib/promscrape/targets_response.qtpl:71
+			if !ts.up {
+//line lib/promscrape/targets_response.qtpl:71
+				qw422016.N().S(`class="alert alert-danger" role="alert"`)
+//line lib/promscrape/targets_response.qtpl:71
+			}
+//line lib/promscrape/targets_response.qtpl:71
+			qw422016.N().S(`> <td><a href="`)
+//line lib/promscrape/targets_response.qtpl:72
+			qw422016.E().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:72
 			qw422016.N().S(`">`)
-//line lib/promscrape/targets_response.qtpl:78
-			qw422016.N().S(ts.endpoint)
-//line lib/promscrape/targets_response.qtpl:78
-			qw422016.N().S(`</a><br></td><td class="state border"><span class="state_indicator">`)
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:72
+			qw422016.E().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:72
+			qw422016.N().S(`</a><br></td> <td>`)
+//line lib/promscrape/targets_response.qtpl:73
 			if ts.up {
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:73
 				qw422016.N().S(`UP`)
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:73
 			} else {
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:73
 				qw422016.N().S(`DOWN`)
-//line lib/promscrape/targets_response.qtpl:81
+//line lib/promscrape/targets_response.qtpl:73
 			}
-//line lib/promscrape/targets_response.qtpl:81
-			qw422016.N().S(`</span></td><td class="labels border", title="Original`)
-//line lib/promscrape/targets_response.qtpl:83
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:83
-			qw422016.N().S(`labels:`)
-//line lib/promscrape/targets_response.qtpl:83
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:83
+//line lib/promscrape/targets_response.qtpl:73
+			qw422016.N().S(`</td> <td title="Original labels: `)
+//line lib/promscrape/targets_response.qtpl:74
 			streamformatLabel(qw422016, ts.originalLabels)
-//line lib/promscrape/targets_response.qtpl:83
-			qw422016.N().S(`">`)
-//line lib/promscrape/targets_response.qtpl:84
+//line lib/promscrape/targets_response.qtpl:74
+			qw422016.N().S(`"> `)
+//line lib/promscrape/targets_response.qtpl:75
 			streamformatLabel(qw422016, ts.labels)
-//line lib/promscrape/targets_response.qtpl:84
-			qw422016.N().S(`</td><td class="last-scrape border">`)
-//line lib/promscrape/targets_response.qtpl:86
-			qw422016.E().S(ts.lastScrapeTime.String())
-//line lib/promscrape/targets_response.qtpl:86
-			qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:86
-			qw422016.N().S(`ago</td><td class="scrape-duration border">`)
-//line lib/promscrape/targets_response.qtpl:87
-			qw422016.E().S(ts.scrapeDuration.String())
-//line lib/promscrape/targets_response.qtpl:87
-			qw422016.N().S(`</td><td class="errors border"><span class="alert alert-danger state_indicator">`)
-//line lib/promscrape/targets_response.qtpl:88
-			qw422016.N().S(ts.error)
-//line lib/promscrape/targets_response.qtpl:88
-			qw422016.N().S(`</span></td></tr>`)
-//line lib/promscrape/targets_response.qtpl:90
+//line lib/promscrape/targets_response.qtpl:75
+			qw422016.N().S(` </td> <td>`)
+//line lib/promscrape/targets_response.qtpl:77
+			qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
+//line lib/promscrape/targets_response.qtpl:77
+			qw422016.N().S(`s ago</td> <td>`)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3)
+//line lib/promscrape/targets_response.qtpl:78
+			qw422016.N().S(`s</td> <td>`)
+//line lib/promscrape/targets_response.qtpl:79
+			qw422016.E().S(ts.error)
+//line lib/promscrape/targets_response.qtpl:79
+			qw422016.N().S(`</td> </tr> `)
+//line lib/promscrape/targets_response.qtpl:81
 		}
-//line lib/promscrape/targets_response.qtpl:90
-		qw422016.N().S(`</tbody></table></div>`)
-//line lib/promscrape/targets_response.qtpl:94
+//line lib/promscrape/targets_response.qtpl:81
+		qw422016.N().S(` </tbody> </table> </div> `)
+//line lib/promscrape/targets_response.qtpl:85
 	}
-//line lib/promscrape/targets_response.qtpl:94
-	qw422016.N().S(`</body></html>`)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:85
+	qw422016.N().S(` </body> </html> `)
+//line lib/promscrape/targets_response.qtpl:88
 }
 
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 }
 
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string {
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 	return qs422016
-//line lib/promscrape/targets_response.qtpl:97
+//line lib/promscrape/targets_response.qtpl:88
 }
 
-//line lib/promscrape/targets_response.qtpl:99
+//line lib/promscrape/targets_response.qtpl:90
 func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:100
+//line lib/promscrape/targets_response.qtpl:90
+	qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:91
 	for _, label := range labels {
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:91
 		qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:92
 		qw422016.E().S(label.Name)
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:92
 		qw422016.N().S(`=`)
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:92
 		qw422016.E().Q(label.Value)
-//line lib/promscrape/targets_response.qtpl:101
+//line lib/promscrape/targets_response.qtpl:92
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:92
 		qw422016.N().S(` `)
-//line lib/promscrape/targets_response.qtpl:102
+//line lib/promscrape/targets_response.qtpl:92
+		qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:93
 	}
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:93
+	qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:94
 }
 
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) {
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	streamformatLabel(qw422016, labels)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 }
 
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 func formatLabel(labels []prompbmarshal.Label) string {
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	writeformatLabel(qb422016, labels)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	qs422016 := string(qb422016.B)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 	return qs422016
-//line lib/promscrape/targets_response.qtpl:103
+//line lib/promscrape/targets_response.qtpl:94
 }


Test patch:
diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go
index 0539986e3..13dd21731 100644
--- a/app/vmalert/config/config_test.go
+++ b/app/vmalert/config/config_test.go
@@ -7,8 +7,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"gopkg.in/yaml.v2"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
 
 func TestMain(m *testing.M) {
@@ -42,7 +43,7 @@ func TestParseBad(t *testing.T) {
 		},
 		{
 			[]string{"testdata/dir/rules2-bad.rules"},
-			"function \"value\" not defined",
+			"function \"unknown\" not defined",
 		},
 		{
 			[]string{"testdata/dir/rules3-bad.rules"},
@@ -137,12 +138,14 @@ func TestGroup_Validate(t *testing.T) {
 						Alert: "alert",
 						Expr:  "up == 1",
 						Labels: map[string]string{
-							"summary": "{{ value|query }}",
+							"summary": `
+{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
+  {{ . | first | value | humanize1024 }}B
+{{ end }}`,
 						},
 					},
 				},
 			},
-			expErr:              "error parsing annotation",
 			validateAnnotations: true,
 		},
 		{
diff --git a/app/vmalert/config/testdata/dir/rules2-bad.rules b/app/vmalert/config/testdata/dir/rules2-bad.rules
index 57ed21357..903921921 100644
--- a/app/vmalert/config/testdata/dir/rules2-bad.rules
+++ b/app/vmalert/config/testdata/dir/rules2-bad.rules
@@ -6,6 +6,6 @@ groups:
         expr: vm_rows > 0
         labels:
           label: bar
-          summary: "{{ value|query }}"
+          summary: "{{ unknown|query }}"
         annotations:
           description: "{{$labels}}"
diff --git a/app/vmalert/config/testdata/rules2-good.rules b/app/vmalert/config/testdata/rules2-good.rules
index 3a015d3eb..a3ce92342 100644
--- a/app/vmalert/config/testdata/rules2-good.rules
+++ b/app/vmalert/config/testdata/rules2-good.rules
@@ -7,11 +7,18 @@ groups:
         expr: sum(vm_tcplistener_conns) by(instance) > 1
         for: 3m
         annotations:
-          summary: "Too high connection number for {{$labels.instance}}"
+          summary: Too high connection number for {{$labels.instance}}
+            {{ with printf "sum(vm_tcplistener_conns{instance=%q})" .Labels.instance | query }}
+              {{ . | first | value }}
+            {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
       - alert: ExampleAlertAlwaysFiring
         expr: sum by(job)
           (up == 1)
+        annotations:
+          summary: Instances up {{ range query "up" }}
+            {{ . | label "instance" }}
+            {{ end }}
       - record: handler:requests:rate5m
         expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
         labels:
diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go
index d8b98dcdd..093817abf 100644
--- a/app/vmalert/group_test.go
+++ b/app/vmalert/group_test.go
@@ -167,7 +167,7 @@ func TestGroupStart(t *testing.T) {
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
 	r := g.Rules[0].(*AlertingRule)
-	alert1, err := r.newAlert(m1, time.Now())
+	alert1, err := r.newAlert(m1, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -179,7 +179,7 @@ func TestGroupStart(t *testing.T) {
 	alert1.Labels["host"] = inst1
 	alert1.ID = hash(m1)
 
-	alert2, err := r.newAlert(m2, time.Now())
+	alert2, err := r.newAlert(m2, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go
index f663be1c0..bc4e7d2c3 100644
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@@ -2,6 +2,8 @@ package notifier
 
 import (
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
@@ -60,11 +62,41 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"exprEscapedPath":  "vm_rows%7B%5C%22label%5C%22=%5C%22bar%5C%22%7D%3E0",
 			},
 		},
+		{
+			name:  "query",
+			alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
+			annotations: map[string]string{
+				"summary": `{{ query "foo" | first | value }}`,
+				"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+			},
+			expTpl: map[string]string{
+				"summary": "1",
+				"desc":    "bar 1;garply 2;",
+			},
+		},
 	}
 
+	qFn := func(q string) ([]datasource.Metric, error) {
+		return []datasource.Metric{
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "bar"},
+					{Name: "baz", Value: "qux"},
+				},
+				Value: 1,
+			},
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "garply"},
+					{Name: "baz", Value: "fred"},
+				},
+				Value: 2,
+			},
+		}, nil
+	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			tpl, err := tc.alert.ExecTemplate(tc.annotations)
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.annotations)
 			if err != nil {
 				t.Fatal(err)
 			}
diff --git a/lib/flagutil/array_test.go b/lib/flagutil/array_test.go
index a29e59e17..7e27ab2ff 100644
--- a/lib/flagutil/array_test.go
+++ b/lib/flagutil/array_test.go
@@ -5,13 +5,21 @@ import (
 	"os"
 	"reflect"
 	"testing"
+	"time"
 )
 
-var fooFlag Array
+var (
+	fooFlag         Array
+	fooFlagDuration ArrayDuration
+	fooFlagBool     ArrayBool
+)
 
 func init() {
-	os.Args = append(os.Args, "--fooFlag=foo", "--fooFlag=bar")
+	os.Args = append(os.Args, "--fooFlag=foo", "--fooFlag=bar", "--fooFlagDuration=10s", "--fooFlagDuration=5m")
+	os.Args = append(os.Args, "--fooFlagBool=true", "--fooFlagBool=false,true", "--fooFlagBool")
 	flag.Var(&fooFlag, "fooFlag", "test")
+	flag.Var(&fooFlagDuration, "fooFlagDuration", "test")
+	flag.Var(&fooFlagBool, "fooFlagBool", "test")
 }
 
 func TestMain(m *testing.M) {
@@ -91,3 +99,123 @@ func TestArrayString(t *testing.T) {
 	f(`", foo","b\"ar",`)
 	f(`,"\nfoo\\",bar`)
 }
+
+func TestArrayDuration(t *testing.T) {
+	expected := map[time.Duration]struct{}{
+		time.Second * 10: {},
+		time.Minute * 5:  {},
+	}
+	if len(expected) != len(fooFlagDuration) {
+		t.Errorf("len array flag (%d) is not equal to %d", len(fooFlag), len(expected))
+	}
+	for _, i := range fooFlagDuration {
+		if _, ok := expected[i]; !ok {
+			t.Errorf("unexpected item in array %v", i)
+		}
+	}
+}
+
+func TestArrayDurationSet(t *testing.T) {
+	f := func(s string, expectedValues []time.Duration) {
+		t.Helper()
+		var a ArrayDuration
+		_ = a.Set(s)
+		if !reflect.DeepEqual([]time.Duration(a), expectedValues) {
+			t.Fatalf("unexpected values parsed;\ngot\n%q\nwant\n%q", a, expectedValues)
+		}
+	}
+	f("", nil)
+	f(`1m`, []time.Duration{time.Minute})
+	f(`5m,1s,1h`, []time.Duration{time.Minute * 5, time.Second, time.Hour})
+}
+
+func TestArrayDurationGetOptionalArg(t *testing.T) {
+	f := func(s string, argIdx int, expectedValue time.Duration, defaultValue time.Duration) {
+		t.Helper()
+		var a ArrayDuration
+		_ = a.Set(s)
+		v := a.GetOptionalArgOrDefault(argIdx, defaultValue)
+		if v != expectedValue {
+			t.Fatalf("unexpected value; got %q; want %q", v, expectedValue)
+		}
+	}
+	f("", 0, time.Second, time.Second)
+	f("", 1, time.Minute, time.Minute)
+	f("10s,1m", 1, time.Minute, time.Minute)
+	f("10s", 3, time.Second*10, time.Minute)
+}
+
+func TestArrayDurationString(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		var a ArrayDuration
+		_ = a.Set(s)
+		result := a.String()
+		if result != s {
+			t.Fatalf("unexpected string;\ngot\n%s\nwant\n%s", result, s)
+		}
+	}
+	f("")
+	f("10s,1m0s")
+	f("5m0s,1s")
+}
+
+func TestArrayBool(t *testing.T) {
+	expected := []bool{
+		true, false, true, true,
+	}
+	if len(expected) != len(fooFlagBool) {
+		t.Errorf("len array flag (%d) is not equal to %d", len(fooFlag), len(expected))
+	}
+	for i, v := range fooFlagBool {
+		if v != expected[i] {
+			t.Errorf("unexpected item in array index=%v,value=%v,want=%v", i, v, expected[i])
+		}
+	}
+}
+
+func TestArrayBoolSet(t *testing.T) {
+	f := func(s string, expectedValues []bool) {
+		t.Helper()
+		var a ArrayBool
+		_ = a.Set(s)
+		if !reflect.DeepEqual([]bool(a), expectedValues) {
+			t.Fatalf("unexpected values parsed;\ngot\n%v\nwant\n%v", a, expectedValues)
+		}
+	}
+	f("", nil)
+	f(`true`, []bool{true})
+	f(`false,True,False`, []bool{false, true, false})
+}
+
+func TestArrayBoolGetOptionalArg(t *testing.T) {
+	f := func(s string, argIdx int, expectedValue bool) {
+		t.Helper()
+		var a ArrayBool
+		_ = a.Set(s)
+		v := a.GetOptionalArg(argIdx)
+		if v != expectedValue {
+			t.Fatalf("unexpected value; got %v; want %v", v, expectedValue)
+		}
+	}
+	f("", 0, false)
+	f("", 1, false)
+	f("true,true,false", 1, true)
+	f("true", 2, true)
+}
+
+func TestArrayBoolString(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		var a ArrayBool
+		_ = a.Set(s)
+		result := a.String()
+		if result != s {
+			t.Fatalf("unexpected string;\ngot\n%s\nwant\n%s", result, s)
+		}
+	}
+	f("")
+	f("true")
+	f("true,false")
+	f("false,true")
+}