Repository: VictoriaMetrics/VictoriaMetrics
Pull request #2320 (2022-03-15 06:57:10)

Vmalert compliance improvements
See the compliance doc here https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md


Patch:
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 760413266..8b1b5f3a6 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -526,6 +526,8 @@ The shortlist of configuration flags is the following:
     	Optional OAuth2 tokenURL to use for -datasource.url.
   -datasource.queryStep duration
     	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+  -datasource.queryTimeAlignment
+        Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
     	Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
   -datasource.tlsCAFile string
diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go
index 9cd86fd21..34d59ea19 100644
--- a/app/vmalert/alerting.go
+++ b/app/vmalert/alerting.go
@@ -38,6 +38,8 @@ type AlertingRule struct {
 	alerts map[uint64]*notifier.Alert
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -203,12 +205,14 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
 func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
 	qMetrics, err := ar.q.Query(ctx, ar.Expr)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
 	ar.lastExecError = err
-	ar.lastExecTime = time.Now()
 	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
@@ -386,31 +390,48 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// RuleAPI returns Rule representation in form
-// of APIAlertingRule
-func (ar *AlertingRule) RuleAPI() APIAlertingRule {
-	var lastErr string
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
 	if ar.lastExecError != nil {
-		lastErr = ar.lastExecError.Error()
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
 	}
-	return APIAlertingRule{
-		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", ar.ID()),
-		GroupID:     fmt.Sprintf("%d", ar.GroupID),
-		Type:        ar.Type.String(),
-		Name:        ar.Name,
-		Expression:  ar.Expr,
-		For:         ar.For.String(),
-		LastError:   lastErr,
-		LastSamples: ar.lastExecSamples,
-		LastExec:    ar.lastExecTime,
-		Labels:      ar.Labels,
-		Annotations: ar.Annotations,
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
 	}
+	return r
 }
 
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
 	ar.mu.RLock()
 	for _, a := range ar.alerts {
diff --git a/app/vmalert/datasource/init.go b/app/vmalert/datasource/init.go
index f36059f51..004a9b36d 100644
--- a/app/vmalert/datasource/init.go
+++ b/app/vmalert/datasource/init.go
@@ -38,6 +38,8 @@ var (
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
 		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
 		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
diff --git a/app/vmalert/datasource/vm_prom_api.go b/app/vmalert/datasource/vm_prom_api.go
index 3bf358a24..5411fa181 100644
--- a/app/vmalert/datasource/vm_prom_api.go
+++ b/app/vmalert/datasource/vm_prom_api.go
@@ -125,7 +125,7 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
 	}
-	if s.evaluationInterval > 0 {
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
diff --git a/app/vmalert/group.go b/app/vmalert/group.go
index addcfa3ff..bedf1de67 100644
--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -19,14 +19,15 @@ import (
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu          sync.RWMutex
-	Name        string
-	File        string
-	Rules       []Rule
-	Type        datasource.Type
-	Interval    time.Duration
-	Concurrency int
-	Checksum    string
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           datasource.Type
+	Interval       time.Duration
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
 
 	Labels map[string]string
 	Params url.Values
@@ -283,6 +284,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 						logger.Errorf("group %q: %s", g.Name, err)
 					}
 				}
+				g.LastEvaluation = iterationStart
 			}
 			g.metrics.iterationDuration.UpdateDuration(iterationStart)
 		}
@@ -347,6 +349,7 @@ var (
 func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
 	execTotal.Inc()
 
+	now := time.Now()
 	tss, err := rule.Exec(ctx)
 	if err != nil {
 		execErrors.Inc()
@@ -371,12 +374,12 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur
 	for _, a := range ar.alerts {
 		switch a.State {
 		case notifier.StateFiring:
-			a.End = time.Now().Add(resolveDuration)
+			a.End = now.Add(resolveDuration)
 			alerts = append(alerts, *a)
 		case notifier.StateInactive:
 			// set End to execStart to notify
 			// that it was just resolved
-			a.End = time.Now()
+			a.End = now
 			alerts = append(alerts, *a)
 		}
 	}
diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go
index 24f872faf..7152fd258 100644
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -163,21 +163,17 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:        g.Name,
-		Type:        g.Type.String(),
-		File:        g.File,
-		Interval:    g.Interval.String(),
-		Concurrency: g.Concurrency,
-		Params:      urlValuesToStrings(g.Params),
-		Labels:      g.Labels,
+		Name:           g.Name,
+		Type:           g.Type.String(),
+		File:           g.File,
+		Interval:       g.Interval.Seconds(),
+		LastEvaluation: g.LastEvaluation,
+		Concurrency:    g.Concurrency,
+		Params:         urlValuesToStrings(g.Params),
+		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
-		switch v := r.(type) {
-		case *AlertingRule:
-			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
-		case *RecordingRule:
-			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
-		}
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return ag
 }
diff --git a/app/vmalert/notifier/template_func.go b/app/vmalert/notifier/template_func.go
index 6cd3c0afe..c217bc6ff 100644
--- a/app/vmalert/notifier/template_func.go
+++ b/app/vmalert/notifier/template_func.go
@@ -20,6 +20,7 @@ import (
 	"net"
 	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"time"
 
@@ -282,6 +283,14 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return m.Labels[label]
 		},
 
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
 		// value returns the value of the given metric.
 		// usually used alongside with `query` template function.
 		"value": func(m metric) float64 {
diff --git a/app/vmalert/recording.go b/app/vmalert/recording.go
index d05ffc17d..2ae840290 100644
--- a/app/vmalert/recording.go
+++ b/app/vmalert/recording.go
@@ -31,6 +31,8 @@ type RecordingRule struct {
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -123,11 +125,13 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 
 // Exec executes RecordingRule expression via the given Querier.
 func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
 	qMetrics, err := rr.q.Query(ctx, rr.Expr)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
 
-	rr.lastExecTime = time.Now()
+	rr.lastExecTime = start
+	rr.lastExecDuration = time.Since(start)
 	rr.lastExecError = err
 	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
@@ -193,23 +197,27 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	return nil
 }
 
-// RuleAPI returns Rule representation in form
-// of APIRecordingRule
-func (rr *RecordingRule) RuleAPI() APIRecordingRule {
-	var lastErr string
-	if rr.lastExecError != nil {
-		lastErr = rr.lastExecError.Error()
-	}
-	return APIRecordingRule{
+// ToAPI returns Rule's representation in form
+// of APIRule
+func (rr *RecordingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "recording",
+		DatasourceType: rr.Type.String(),
+		Name:           rr.Name,
+		Query:          rr.Expr,
+		Labels:         rr.Labels,
+		LastEvaluation: rr.lastExecTime,
+		EvaluationTime: rr.lastExecDuration.Seconds(),
+		Health:         "ok",
+		LastSamples:    rr.lastExecSamples,
 		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", rr.ID()),
-		GroupID:     fmt.Sprintf("%d", rr.GroupID),
-		Name:        rr.Name,
-		Type:        rr.Type.String(),
-		Expression:  rr.Expr,
-		LastError:   lastErr,
-		LastSamples: rr.lastExecSamples,
-		LastExec:    rr.lastExecTime,
-		Labels:      rr.Labels,
+		ID:      fmt.Sprintf("%d", rr.ID()),
+		GroupID: fmt.Sprintf("%d", rr.GroupID),
+	}
+
+	if rr.lastExecError != nil {
+		r.LastError = rr.lastExecError.Error()
+		r.Health = "err"
 	}
+	return r
 }
diff --git a/app/vmalert/rule.go b/app/vmalert/rule.go
index 6281bc09a..5157d6b09 100644
--- a/app/vmalert/rule.go
+++ b/app/vmalert/rule.go
@@ -21,6 +21,8 @@ type Rule interface {
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error
+	// ToAPI converts Rule into APIRule
+	ToAPI() APIRule
 	// Close performs the shutdown procedures for rule
 	// such as metrics unregister
 	Close()
diff --git a/app/vmalert/web.go b/app/vmalert/web.go
index 51b9e44de..ef59a89f0 100644
--- a/app/vmalert/web.go
+++ b/app/vmalert/web.go
@@ -29,7 +29,7 @@ func initLinks() {
 		pathPrefix = "/"
 	}
 	apiLinks = [][2]string{
-		{path.Join(pathPrefix, "api/v1/groups"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/rules"), "list all loaded groups and rules"},
 		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
 		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
 		{path.Join(pathPrefix, "flags"), "command-line flags"},
@@ -75,7 +75,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/notifiers":
 		WriteListTargets(w, notifier.GetTargets())
 		return true
-	case "/api/v1/groups":
+	case "/api/v1/rules":
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
@@ -127,10 +127,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 type listGroupsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Groups []APIGroup `json:"groups"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
 func (rh *requestHandler) groups() []APIGroup {
@@ -149,6 +149,7 @@ func (rh *requestHandler) groups() []APIGroup {
 
 	return groups
 }
+
 func (rh *requestHandler) listGroups() ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
 	lr.Data.Groups = rh.groups()
@@ -163,10 +164,10 @@ func (rh *requestHandler) listGroups() ([]byte, error) {
 }
 
 type listAlertsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Alerts []*APIAlert `json:"alerts"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
 func (rh *requestHandler) groupAlerts() []GroupAlerts {
@@ -181,7 +182,7 @@ func (rh *requestHandler) groupAlerts() []GroupAlerts {
 			if !ok {
 				continue
 			}
-			alerts = append(alerts, a.AlertsAPI()...)
+			alerts = append(alerts, a.AlertsToAPI()...)
 		}
 		if len(alerts) > 0 {
 			groupAlerts = append(groupAlerts, GroupAlerts{
@@ -204,7 +205,7 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 			if !ok {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
 		}
 	}
 
diff --git a/app/vmalert/web.qtpl b/app/vmalert/web.qtpl
index a93cc4b29..89b50b5d8 100644
--- a/app/vmalert/web.qtpl
+++ b/app/vmalert/web.qtpl
@@ -31,14 +31,7 @@
             rOk := make(map[string]int)
             rNotOk := make(map[string]int)
             for _, g := range groups {
-                for _, r := range g.AlertingRules{
-                    if r.LastError != "" {
-                        rNotOk[g.Name]++
-                    } else {
-                        rOk[g.Name]++
-                    }
-                }
-                 for _, r := range g.RecordingRules{
+                for _, r := range g.Rules {
                     if r.LastError != "" {
                         rNotOk[g.Name]++
                     } else {
@@ -52,7 +45,7 @@
         {% for _, g := range groups  %}
               <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
-                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%s g.Interval %})</a>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
                  {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
                 <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>
@@ -75,34 +68,24 @@
                         </tr>
                     </thead>
                     <tbody>
-                    {% for _, ar := range g.AlertingRules %}
-                        <tr{% if ar.LastError != "" %} class="alert-danger"{% endif %}>
-                            <td>
-                                <b>alert:</b> {%s ar.Name %} (for: {%v ar.For %})<br>
-                                <code><pre>{%s ar.Expression %}</pre></code><br>
-                                {% if len(ar.Labels) > 0 %} <b>Labels:</b>{% endif %}
-                                {% for k, v := range ar.Labels %}
-                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
-                                {% endfor %}
-                            </td>
-                            <td><div class="error-cell">{%s ar.LastError %}</div></td>
-                            <td>{%d ar.LastSamples %}</td>
-                            <td>{%f.3 time.Since(ar.LastExec).Seconds() %}s ago</td>
-                        </tr>
-                    {% endfor %}
-                    {% for _, rr := range g.RecordingRules  %}
-                        <tr>
+                    {% for _, r := range g.Rules %}
+                        <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
                             <td>
-                                <b>record:</b> {%s rr.Name %}<br>
-                                <code><pre>{%s rr.Expression %}</pre></code>
-                                {% if len(rr.Labels) > 0 %} <b>Labels:</b>{% endif %}
-                                {% for k, v := range rr.Labels %}
+                                {% if r.Type == "alerting" %}
+                                <b>alert:</b> (for: {%v r.Duration %})
+                                {% else %}
+                                <b>record:</b> {%s r.Name %}
+                                {% endif %}
+                                <br>
+                                <code><pre>{%s r.Query %}</pre></code><br>
+                                {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range r.Labels %}
                                         <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
                                 {% endfor %}
                             </td>
-                            <td><div class="error-cell">{%s rr.LastError %}</div></td>
-                            <td>{%d rr.LastSamples %}</td>
-                            <td>{%f.3 time.Since(rr.LastExec).Seconds() %}s ago</td>
+                            <td><div class="error-cell">{%s r.LastError %}</div></td>
+                            <td>{%d r.LastSamples %}</td>
+                            <td>{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
                         </tr>
                     {% endfor %}
                  </tbody>
diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go
index b73d7c4bd..dd6f4daf2 100644
--- a/app/vmalert/web.qtpl.go
+++ b/app/vmalert/web.qtpl.go
@@ -122,14 +122,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) {
 		rOk := make(map[string]int)
 		rNotOk := make(map[string]int)
 		for _, g := range groups {
-			for _, r := range g.AlertingRules {
-				if r.LastError != "" {
-					rNotOk[g.Name]++
-				} else {
-					rOk[g.Name]++
-				}
-			}
-			for _, r := range g.RecordingRules {
+			for _, r := range g.Rules {
 				if r.LastError != "" {
 					rNotOk[g.Name]++
 				} else {
@@ -138,111 +131,111 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) {
 			}
 		}
 
-//line app/vmalert/web.qtpl:49
+//line app/vmalert/web.qtpl:42
 		qw422016.N().S(`
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
         `)
-//line app/vmalert/web.qtpl:52
+//line app/vmalert/web.qtpl:45
 		for _, g := range groups {
-//line app/vmalert/web.qtpl:52
+//line app/vmalert/web.qtpl:45
 			qw422016.N().S(`
               <div class="group-heading`)
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 			if rNotOk[g.Name] > 0 {
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 				qw422016.N().S(` alert-danger`)
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 			}
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 			qw422016.N().S(`"  data-bs-target="rules-`)
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:53
+//line app/vmalert/web.qtpl:46
 			qw422016.N().S(`">
                 <span class="anchor" id="group-`)
-//line app/vmalert/web.qtpl:54
+//line app/vmalert/web.qtpl:47
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:54
+//line app/vmalert/web.qtpl:47
 			qw422016.N().S(`"></span>
                 <a href="#group-`)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			qw422016.E().S(g.Name)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			if g.Type != "prometheus" {
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 				qw422016.N().S(` (`)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 				qw422016.E().S(g.Type)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 				qw422016.N().S(`)`)
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			}
-//line app/vmalert/web.qtpl:55
+//line app/vmalert/web.qtpl:48
 			qw422016.N().S(` (every `)
-//line app/vmalert/web.qtpl:55
-			qw422016.E().S(g.Interval)
-//line app/vmalert/web.qtpl:55
-			qw422016.N().S(`)</a>
+//line app/vmalert/web.qtpl:48
+			qw422016.N().FPrec(g.Interval, 0)
+//line app/vmalert/web.qtpl:48
+			qw422016.N().S(`s)</a>
                  `)
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 			if rNotOk[g.Name] > 0 {
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 				qw422016.N().S(`<span class="badge bg-danger" title="Number of rules with status Error">`)
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 				qw422016.N().D(rNotOk[g.Name])
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 				qw422016.N().S(`</span> `)
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 			}
-//line app/vmalert/web.qtpl:56
+//line app/vmalert/web.qtpl:49
 			qw422016.N().S(`
                 <span class="badge bg-success" title="Number of rules withs status Ok">`)
-//line app/vmalert/web.qtpl:57
+//line app/vmalert/web.qtpl:50
 			qw422016.N().D(rOk[g.Name])
-//line app/vmalert/web.qtpl:57
+//line app/vmalert/web.qtpl:50
 			qw422016.N().S(`</span>
                 <p class="fs-6 fw-lighter">`)
-//line app/vmalert/web.qtpl:58
+//line app/vmalert/web.qtpl:51
 			qw422016.E().S(g.File)
-//line app/vmalert/web.qtpl:58
+//line app/vmalert/web.qtpl:51
 			qw422016.N().S(`</p>
                 `)
-//line app/vmalert/web.qtpl:59
+//line app/vmalert/web.qtpl:52
 			if len(g.Params) > 0 {
-//line app/vmalert/web.qtpl:59
+//line app/vmalert/web.qtpl:52
 				qw422016.N().S(`
                     <div class="fs-6 fw-lighter">Extra params
                     `)
-//line app/vmalert/web.qtpl:61
+//line app/vmalert/web.qtpl:54
 				for _, param := range g.Params {
-//line app/vmalert/web.qtpl:61
+//line app/vmalert/web.qtpl:54
 					qw422016.N().S(`
                             <span class="float-left badge bg-primary">`)
-//line app/vmalert/web.qtpl:62
+//line app/vmalert/web.qtpl:55
 					qw422016.E().S(param)
-//line app/vmalert/web.qtpl:62
+//line app/vmalert/web.qtpl:55
 					qw422016.N().S(`</span>
                     `)
-//line app/vmalert/web.qtpl:63
+//line app/vmalert/web.qtpl:56
 				}
-//line app/vmalert/web.qtpl:63
+//line app/vmalert/web.qtpl:56
 				qw422016.N().S(`
                     </div>
                 `)
-//line app/vmalert/web.qtpl:65
+//line app/vmalert/web.qtpl:58
 			}
-//line app/vmalert/web.qtpl:65
+//line app/vmalert/web.qtpl:58
 			qw422016.N().S(`
             </div>
             <div class="collapse" id="rules-`)
-//line app/vmalert/web.qtpl:67
+//line app/vmalert/web.qtpl:60
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:67
+//line app/vmalert/web.qtpl:60
 			qw422016.N().S(`">
                 <table class="table table-striped table-hover table-sm">
                     <thead>
@@ -255,280 +248,230 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) {
                     </thead>
                     <tbody>
                     `)
-//line app/vmalert/web.qtpl:78
-			for _, ar := range g.AlertingRules {
-//line app/vmalert/web.qtpl:78
+//line app/vmalert/web.qtpl:71
+			for _, r := range g.Rules {
+//line app/vmalert/web.qtpl:71
 				qw422016.N().S(`
                         <tr`)
-//line app/vmalert/web.qtpl:79
-				if ar.LastError != "" {
-//line app/vmalert/web.qtpl:79
+//line app/vmalert/web.qtpl:72
+				if r.LastError != "" {
+//line app/vmalert/web.qtpl:72
 					qw422016.N().S(` class="alert-danger"`)
-//line app/vmalert/web.qtpl:79
+//line app/vmalert/web.qtpl:72
 				}
-//line app/vmalert/web.qtpl:79
+//line app/vmalert/web.qtpl:72
 				qw422016.N().S(`>
                             <td>
-                                <b>alert:</b> `)
-//line app/vmalert/web.qtpl:81
-				qw422016.E().S(ar.Name)
-//line app/vmalert/web.qtpl:81
-				qw422016.N().S(` (for: `)
-//line app/vmalert/web.qtpl:81
-				qw422016.E().V(ar.For)
-//line app/vmalert/web.qtpl:81
-				qw422016.N().S(`)<br>
-                                <code><pre>`)
-//line app/vmalert/web.qtpl:82
-				qw422016.E().S(ar.Expression)
-//line app/vmalert/web.qtpl:82
-				qw422016.N().S(`</pre></code><br>
                                 `)
-//line app/vmalert/web.qtpl:83
-				if len(ar.Labels) > 0 {
-//line app/vmalert/web.qtpl:83
-					qw422016.N().S(` <b>Labels:</b>`)
-//line app/vmalert/web.qtpl:83
-				}
-//line app/vmalert/web.qtpl:83
-				qw422016.N().S(`
+//line app/vmalert/web.qtpl:74
+				if r.Type == "alerting" {
+//line app/vmalert/web.qtpl:74
+					qw422016.N().S(`
+                                <b>alert:</b> (for: `)
+//line app/vmalert/web.qtpl:75
+					qw422016.E().V(r.Duration)
+//line app/vmalert/web.qtpl:75
+					qw422016.N().S(`)
                                 `)
-//line app/vmalert/web.qtpl:84
-				for k, v := range ar.Labels {
-//line app/vmalert/web.qtpl:84
+//line app/vmalert/web.qtpl:76
+				} else {
+//line app/vmalert/web.qtpl:76
+					qw422016.N().S(`
+                                <b>record:</b> `)
+//line app/vmalert/web.qtpl:77
+					qw422016.E().S(r.Name)
+//line app/vmalert/web.qtpl:77
 					qw422016.N().S(`
-                                        <span class="ms-1 badge bg-primary">`)
-//line app/vmalert/web.qtpl:85
-					qw422016.E().S(k)
-//line app/vmalert/web.qtpl:85
-					qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:85
-					qw422016.E().S(v)
-//line app/vmalert/web.qtpl:85
-					qw422016.N().S(`</span>
                                 `)
-//line app/vmalert/web.qtpl:86
+//line app/vmalert/web.qtpl:78
 				}
-//line app/vmalert/web.qtpl:86
-				qw422016.N().S(`
-                            </td>
-                            <td><div class="error-cell">`)
-//line app/vmalert/web.qtpl:88
-				qw422016.E().S(ar.LastError)
-//line app/vmalert/web.qtpl:88
-				qw422016.N().S(`</div></td>
-                            <td>`)
-//line app/vmalert/web.qtpl:89
-				qw422016.N().D(ar.LastSamples)
-//line app/vmalert/web.qtpl:89
-				qw422016.N().S(`</td>
-                            <td>`)
-//line app/vmalert/web.qtpl:90
-				qw422016.N().FPrec(time.Since(ar.LastExec).Seconds(), 3)
-//line app/vmalert/web.qtpl:90
-				qw422016.N().S(`s ago</td>
-                        </tr>
-                    `)
-//line app/vmalert/web.qtpl:92
-			}
-//line app/vmalert/web.qtpl:92
-			qw422016.N().S(`
-                    `)
-//line app/vmalert/web.qtpl:93
-			for _, rr := range g.RecordingRules {
-//line app/vmalert/web.qtpl:93
+//line app/vmalert/web.qtpl:78
 				qw422016.N().S(`
-                        <tr>
-                            <td>
-                                <b>record:</b> `)
-//line app/vmalert/web.qtpl:96
-				qw422016.E().S(rr.Name)
-//line app/vmalert/web.qtpl:96
-				qw422016.N().S(`<br>
+                                <br>
                                 <code><pre>`)
-//line app/vmalert/web.qtpl:97
-				qw422016.E().S(rr.Expression)
-//line app/vmalert/web.qtpl:97
-				qw422016.N().S(`</pre></code>
+//line app/vmalert/web.qtpl:80
+				qw422016.E().S(r.Query)
+//line app/vmalert/web.qtpl:80
+				qw422016.N().S(`</pre></code><br>
                                 `)
-//line app/vmalert/web.qtpl:98
-				if len(rr.Labels) > 0 {
-//line app/vmalert/web.qtpl:98
+//line app/vmalert/web.qtpl:81
+				if len(r.Labels) > 0 {
+//line app/vmalert/web.qtpl:81
 					qw422016.N().S(` <b>Labels:</b>`)
-//line app/vmalert/web.qtpl:98
+//line app/vmalert/web.qtpl:81
 				}
-//line app/vmalert/web.qtpl:98
+//line app/vmalert/web.qtpl:81
 				qw422016.N().S(`
                                 `)
-//line app/vmalert/web.qtpl:99
-				for k, v := range rr.Labels {
-//line app/vmalert/web.qtpl:99
+//line app/vmalert/web.qtpl:82
+				for k, v := range r.Labels {
+//line app/vmalert/web.qtpl:82
 					qw422016.N().S(`
                                         <span class="ms-1 badge bg-primary">`)
-//line app/vmalert/web.qtpl:100
+//line app/vmalert/web.qtpl:83
 					qw422016.E().S(k)
-//line app/vmalert/web.qtpl:100
+//line app/vmalert/web.qtpl:83
 					qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:100
+//line app/vmalert/web.qtpl:83
 					qw422016.E().S(v)
-//line app/vmalert/web.qtpl:100
+//line app/vmalert/web.qtpl:83
 					qw422016.N().S(`</span>
                                 `)
-//line app/vmalert/web.qtpl:101
+//line app/vmalert/web.qtpl:84
 				}
-//line app/vmalert/web.qtpl:101
+//line app/vmalert/web.qtpl:84
 				qw422016.N().S(`
                             </td>
                             <td><div class="error-cell">`)
-//line app/vmalert/web.qtpl:103
-				qw422016.E().S(rr.LastError)
-//line app/vmalert/web.qtpl:103
+//line app/vmalert/web.qtpl:86
+				qw422016.E().S(r.LastError)
+//line app/vmalert/web.qtpl:86
 				qw422016.N().S(`</div></td>
                             <td>`)
-//line app/vmalert/web.qtpl:104
-				qw422016.N().D(rr.LastSamples)
-//line app/vmalert/web.qtpl:104
+//line app/vmalert/web.qtpl:87
+				qw422016.N().D(r.LastSamples)
+//line app/vmalert/web.qtpl:87
 				qw422016.N().S(`</td>
                             <td>`)
-//line app/vmalert/web.qtpl:105
-				qw422016.N().FPrec(time.Since(rr.LastExec).Seconds(), 3)
-//line app/vmalert/web.qtpl:105
+//line app/vmalert/web.qtpl:88
+				qw422016.N().FPrec(time.Since(r.LastEvaluation).Seconds(), 3)
+//line app/vmalert/web.qtpl:88
 				qw422016.N().S(`s ago</td>
                         </tr>
                     `)
-//line app/vmalert/web.qtpl:107
+//line app/vmalert/web.qtpl:90
 			}
-//line app/vmalert/web.qtpl:107
+//line app/vmalert/web.qtpl:90
 			qw422016.N().S(`
                  </tbody>
                 </table>
             </div>
         `)
-//line app/vmalert/web.qtpl:111
+//line app/vmalert/web.qtpl:94
 		}
-//line app/vmalert/web.qtpl:111
+//line app/vmalert/web.qtpl:94
 		qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:113
+//line app/vmalert/web.qtpl:96
 	} else {
-//line app/vmalert/web.qtpl:113
+//line app/vmalert/web.qtpl:96
 		qw422016.N().S(`
         <div>
             <p>No items...</p>
         </div>
     `)
-//line app/vmalert/web.qtpl:117
+//line app/vmalert/web.qtpl:100
 	}
-//line app/vmalert/web.qtpl:117
+//line app/vmalert/web.qtpl:100
 	qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:119
+//line app/vmalert/web.qtpl:102
 	tpl.StreamFooter(qw422016)
-//line app/vmalert/web.qtpl:119
+//line app/vmalert/web.qtpl:102
 	qw422016.N().S(`
 
 `)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 }
 
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 func WriteListGroups(qq422016 qtio422016.Writer, groups []APIGroup) {
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	StreamListGroups(qw422016, groups)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 }
 
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 func ListGroups(groups []APIGroup) string {
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	WriteListGroups(qb422016, groups)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 	return qs422016
-//line app/vmalert/web.qtpl:121
+//line app/vmalert/web.qtpl:104
 }
 
-//line app/vmalert/web.qtpl:124
+//line app/vmalert/web.qtpl:107
 func StreamListAlerts(qw422016 *qt422016.Writer, pathPrefix string, groupAlerts []GroupAlerts) {
-//line app/vmalert/web.qtpl:124
+//line app/vmalert/web.qtpl:107
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:125
+//line app/vmalert/web.qtpl:108
 	tpl.StreamHeader(qw422016, "Alerts", navItems)
-//line app/vmalert/web.qtpl:125
+//line app/vmalert/web.qtpl:108
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:126
+//line app/vmalert/web.qtpl:109
 	if len(groupAlerts) > 0 {
-//line app/vmalert/web.qtpl:126
+//line app/vmalert/web.qtpl:109
 		qw422016.N().S(`
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
          `)
-//line app/vmalert/web.qtpl:129
+//line app/vmalert/web.qtpl:112
 		for _, ga := range groupAlerts {
-//line app/vmalert/web.qtpl:129
+//line app/vmalert/web.qtpl:112
 			qw422016.N().S(`
             `)
-//line app/vmalert/web.qtpl:130
+//line app/vmalert/web.qtpl:113
 			g := ga.Group
 
-//line app/vmalert/web.qtpl:130
+//line app/vmalert/web.qtpl:113
 			qw422016.N().S(`
             <div class="group-heading alert-danger" data-bs-target="rules-`)
-//line app/vmalert/web.qtpl:131
+//line app/vmalert/web.qtpl:114
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:131
+//line app/vmalert/web.qtpl:114
 			qw422016.N().S(`">
                 <span class="anchor" id="group-`)
-//line app/vmalert/web.qtpl:132
+//line app/vmalert/web.qtpl:115
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:132
+//line app/vmalert/web.qtpl:115
 			qw422016.N().S(`"></span>
                 <a href="#group-`)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			qw422016.E().S(g.Name)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			if g.Type != "prometheus" {
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 				qw422016.N().S(` (`)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 				qw422016.E().S(g.Type)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 				qw422016.N().S(`)`)
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			}
-//line app/vmalert/web.qtpl:133
+//line app/vmalert/web.qtpl:116
 			qw422016.N().S(`</a>
                 <span class="badge bg-danger" title="Number of active alerts">`)
-//line app/vmalert/web.qtpl:134
+//line app/vmalert/web.qtpl:117
 			qw422016.N().D(len(ga.Alerts))
-//line app/vmalert/web.qtpl:134
+//line app/vmalert/web.qtpl:117
 			qw422016.N().S(`</span>
                 <br>
                 <p class="fs-6 fw-lighter">`)
-//line app/vmalert/web.qtpl:136
+//line app/vmalert/web.qtpl:119
 			qw422016.E().S(g.File)
-//line app/vmalert/web.qtpl:136
+//line app/vmalert/web.qtpl:119
 			qw422016.N().S(`</p>
             </div>
             `)
-//line app/vmalert/web.qtpl:139
+//line app/vmalert/web.qtpl:122
 			var keys []string
 			alertsByRule := make(map[string][]*APIAlert)
 			for _, alert := range ga.Alerts {
@@ -539,20 +482,20 @@ func StreamListAlerts(qw422016 *qt422016.Writer, pathPrefix string, groupAlerts
 			}
 			sort.Strings(keys)
 
-//line app/vmalert/web.qtpl:148
+//line app/vmalert/web.qtpl:131
 			qw422016.N().S(`
             <div class="collapse" id="rules-`)
-//line app/vmalert/web.qtpl:149
+//line app/vmalert/web.qtpl:132
 			qw422016.E().S(g.ID)
-//line app/vmalert/web.qtpl:149
+//line app/vmalert/web.qtpl:132
 			qw422016.N().S(`">
                 `)
-//line app/vmalert/web.qtpl:150
+//line app/vmalert/web.qtpl:133
 			for _, ruleID := range keys {
-//line app/vmalert/web.qtpl:150
+//line app/vmalert/web.qtpl:133
 				qw422016.N().S(`
                     `)
-//line app/vmalert/web.qtpl:152
+//line app/vmalert/web.qtpl:135
 				defaultAR := alertsByRule[ruleID][0]
 				var labelKeys []string
 				for k := range defaultAR.Labels {
@@ -560,28 +503,28 @@ func StreamListAlerts(qw422016 *qt422016.Writer, pathPrefix string, groupAlerts
 				}
 				sort.Strings(labelKeys)
 
-//line app/vmalert/web.qtpl:158
+//line app/vmalert/web.qtpl:141
 				qw422016.N().S(`
                     <br>
                     <b>alert:</b> `)
-//line app/vmalert/web.qtpl:160
+//line app/vmalert/web.qtpl:143
 				qw422016.E().S(defaultAR.Name)
-//line app/vmalert/web.qtpl:160
+//line app/vmalert/web.qtpl:143
 				qw422016.N().S(` (`)
-//line app/vmalert/web.qtpl:160
+//line app/vmalert/web.qtpl:143
 				qw422016.N().D(len(alertsByRule[ruleID]))
-//line app/vmalert/web.qtpl:160
+//line app/vmalert/web.qtpl:143
 				qw422016.N().S(`)
                      | <span><a target="_blank" href="`)
-//line app/vmalert/web.qtpl:161
+//line app/vmalert/web.qtpl:144
 				qw422016.E().S(defaultAR.SourceLink)
-//line app/vmalert/web.qtpl:161
+//line app/vmalert/web.qtpl:144
 				qw422016.N().S(`">Source</a></span>
                     <br>
                     <b>expr:</b><code><pre>`)
-//line app/vmalert/web.qtpl:163
+//line app/vmalert/web.qtpl:146
 				qw422016.E().S(defaultAR.Expression)
-//line app/vmalert/web.qtpl:163
+//line app/vmalert/web.qtpl:146
 				qw422016.N().S(`</pre></code>
                     <table class="table table-striped table-hover table-sm">
                         <thead>
@@ -595,204 +538,204 @@ func StreamListAlerts(qw422016 *qt422016.Writer, pathPrefix string, groupAlerts
                         </thead>
                         <tbody>
                         `)
-//line app/vmalert/web.qtpl:175
+//line app/vmalert/web.qtpl:158
 				for _, ar := range alertsByRule[ruleID] {
-//line app/vmalert/web.qtpl:175
+//line app/vmalert/web.qtpl:158
 					qw422016.N().S(`
                             <tr>
                                 <td>
                                     `)
-//line app/vmalert/web.qtpl:178
+//line app/vmalert/web.qtpl:161
 					for _, k := range labelKeys {
-//line app/vmalert/web.qtpl:178
+//line app/vmalert/web.qtpl:161
 						qw422016.N().S(`
                                         <span class="ms-1 badge bg-primary">`)
-//line app/vmalert/web.qtpl:179
+//line app/vmalert/web.qtpl:162
 						qw422016.E().S(k)
-//line app/vmalert/web.qtpl:179
+//line app/vmalert/web.qtpl:162
 						qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:179
+//line app/vmalert/web.qtpl:162
 						qw422016.E().S(ar.Labels[k])
-//line app/vmalert/web.qtpl:179
+//line app/vmalert/web.qtpl:162
 						qw422016.N().S(`</span>
                                     `)
-//line app/vmalert/web.qtpl:180
+//line app/vmalert/web.qtpl:163
 					}
-//line app/vmalert/web.qtpl:180
+//line app/vmalert/web.qtpl:163
 					qw422016.N().S(`
                                 </td>
                                 <td>`)
-//line app/vmalert/web.qtpl:182
+//line app/vmalert/web.qtpl:165
 					streambadgeState(qw422016, ar.State)
-//line app/vmalert/web.qtpl:182
+//line app/vmalert/web.qtpl:165
 					qw422016.N().S(`</td>
                                 <td>
                                     `)
-//line app/vmalert/web.qtpl:184
+//line app/vmalert/web.qtpl:167
 					qw422016.E().S(ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00"))
-//line app/vmalert/web.qtpl:184
+//line app/vmalert/web.qtpl:167
 					qw422016.N().S(`
                                     `)
-//line app/vmalert/web.qtpl:185
+//line app/vmalert/web.qtpl:168
 					if ar.Restored {
-//line app/vmalert/web.qtpl:185
+//line app/vmalert/web.qtpl:168
 						streambadgeRestored(qw422016)
-//line app/vmalert/web.qtpl:185
+//line app/vmalert/web.qtpl:168
 					}
-//line app/vmalert/web.qtpl:185
+//line app/vmalert/web.qtpl:168
 					qw422016.N().S(`
                                 </td>
                                 <td>`)
-//line app/vmalert/web.qtpl:187
+//line app/vmalert/web.qtpl:170
 					qw422016.E().S(ar.Value)
-//line app/vmalert/web.qtpl:187
+//line app/vmalert/web.qtpl:170
 					qw422016.N().S(`</td>
                                 <td>
                                     <a href="`)
-//line app/vmalert/web.qtpl:189
+//line app/vmalert/web.qtpl:172
 					qw422016.E().S(path.Join(pathPrefix, g.ID, ar.ID, "status"))
-//line app/vmalert/web.qtpl:189
+//line app/vmalert/web.qtpl:172
 					qw422016.N().S(`">Details</a>
                                 </td>
                             </tr>
                         `)
-//line app/vmalert/web.qtpl:192
+//line app/vmalert/web.qtpl:175
 				}
-//line app/vmalert/web.qtpl:192
+//line app/vmalert/web.qtpl:175
 				qw422016.N().S(`
                      </tbody>
                     </table>
                 `)
-//line app/vmalert/web.qtpl:195
+//line app/vmalert/web.qtpl:178
 			}
-//line app/vmalert/web.qtpl:195
+//line app/vmalert/web.qtpl:178
 			qw422016.N().S(`
             </div>
             <br>
         `)
-//line app/vmalert/web.qtpl:198
+//line app/vmalert/web.qtpl:181
 		}
-//line app/vmalert/web.qtpl:198
+//line app/vmalert/web.qtpl:181
 		qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:200
+//line app/vmalert/web.qtpl:183
 	} else {
-//line app/vmalert/web.qtpl:200
+//line app/vmalert/web.qtpl:183
 		qw422016.N().S(`
         <div>
             <p>No items...</p>
         </div>
     `)
-//line app/vmalert/web.qtpl:204
+//line app/vmalert/web.qtpl:187
 	}
-//line app/vmalert/web.qtpl:204
+//line app/vmalert/web.qtpl:187
 	qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:206
+//line app/vmalert/web.qtpl:189
 	tpl.StreamFooter(qw422016)
-//line app/vmalert/web.qtpl:206
+//line app/vmalert/web.qtpl:189
 	qw422016.N().S(`
 
 `)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 }
 
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 func WriteListAlerts(qq422016 qtio422016.Writer, pathPrefix string, groupAlerts []GroupAlerts) {
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	StreamListAlerts(qw422016, pathPrefix, groupAlerts)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 }
 
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 func ListAlerts(pathPrefix string, groupAlerts []GroupAlerts) string {
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	WriteListAlerts(qb422016, pathPrefix, groupAlerts)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 	return qs422016
-//line app/vmalert/web.qtpl:208
+//line app/vmalert/web.qtpl:191
 }
 
-//line app/vmalert/web.qtpl:210
+//line app/vmalert/web.qtpl:193
 func StreamListTargets(qw422016 *qt422016.Writer, targets map[notifier.TargetType][]notifier.Target) {
-//line app/vmalert/web.qtpl:210
+//line app/vmalert/web.qtpl:193
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:211
+//line app/vmalert/web.qtpl:194
 	tpl.StreamHeader(qw422016, "Notifiers", navItems)
-//line app/vmalert/web.qtpl:211
+//line app/vmalert/web.qtpl:194
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:212
+//line app/vmalert/web.qtpl:195
 	if len(targets) > 0 {
-//line app/vmalert/web.qtpl:212
+//line app/vmalert/web.qtpl:195
 		qw422016.N().S(`
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
 
          `)
-//line app/vmalert/web.qtpl:217
+//line app/vmalert/web.qtpl:200
 		var keys []string
 		for key := range targets {
 			keys = append(keys, string(key))
 		}
 		sort.Strings(keys)
 
-//line app/vmalert/web.qtpl:222
+//line app/vmalert/web.qtpl:205
 		qw422016.N().S(`
 
          `)
-//line app/vmalert/web.qtpl:224
+//line app/vmalert/web.qtpl:207
 		for i := range keys {
-//line app/vmalert/web.qtpl:224
+//line app/vmalert/web.qtpl:207
 			qw422016.N().S(`
            `)
-//line app/vmalert/web.qtpl:225
+//line app/vmalert/web.qtpl:208
 			typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
 			count := len(ns)
 
-//line app/vmalert/web.qtpl:227
+//line app/vmalert/web.qtpl:210
 			qw422016.N().S(`
            <div class="group-heading data-bs-target="rules-`)
-//line app/vmalert/web.qtpl:228
+//line app/vmalert/web.qtpl:211
 			qw422016.E().S(typeK)
-//line app/vmalert/web.qtpl:228
+//line app/vmalert/web.qtpl:211
 			qw422016.N().S(`">
              <span class="anchor" id="notifiers-`)
-//line app/vmalert/web.qtpl:229
+//line app/vmalert/web.qtpl:212
 			qw422016.E().S(typeK)
-//line app/vmalert/web.qtpl:229
+//line app/vmalert/web.qtpl:212
 			qw422016.N().S(`"></span>
              <a href="#notifiers-`)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.E().S(typeK)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.E().S(typeK)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.N().S(` (`)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.N().D(count)
-//line app/vmalert/web.qtpl:230
+//line app/vmalert/web.qtpl:213
 			qw422016.N().S(`)</a>
          </div>
          <div class="collapse show" id="notifiers-`)
-//line app/vmalert/web.qtpl:232
+//line app/vmalert/web.qtpl:215
 			qw422016.E().S(typeK)
-//line app/vmalert/web.qtpl:232
+//line app/vmalert/web.qtpl:215
 			qw422016.N().S(`">
              <table class="table table-striped table-hover table-sm">
                  <thead>
@@ -803,113 +746,113 @@ func StreamListTargets(qw422016 *qt422016.Writer, targets map[notifier.TargetTyp
                  </thead>
                  <tbody>
                  `)
-//line app/vmalert/web.qtpl:241
+//line app/vmalert/web.qtpl:224
 			for _, n := range ns {
-//line app/vmalert/web.qtpl:241
+//line app/vmalert/web.qtpl:224
 				qw422016.N().S(`
                      <tr>
                          <td>
                               `)
-//line app/vmalert/web.qtpl:244
+//line app/vmalert/web.qtpl:227
 				for _, l := range n.Labels {
-//line app/vmalert/web.qtpl:244
+//line app/vmalert/web.qtpl:227
 					qw422016.N().S(`
                                       <span class="ms-1 badge bg-primary">`)
-//line app/vmalert/web.qtpl:245
+//line app/vmalert/web.qtpl:228
 					qw422016.E().S(l.Name)
-//line app/vmalert/web.qtpl:245
+//line app/vmalert/web.qtpl:228
 					qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:245
+//line app/vmalert/web.qtpl:228
 					qw422016.E().S(l.Value)
-//line app/vmalert/web.qtpl:245
+//line app/vmalert/web.qtpl:228
 					qw422016.N().S(`</span>
                               `)
-//line app/vmalert/web.qtpl:246
+//line app/vmalert/web.qtpl:229
 				}
-//line app/vmalert/web.qtpl:246
+//line app/vmalert/web.qtpl:229
 				qw422016.N().S(`
                           </td>
                          <td>`)
-//line app/vmalert/web.qtpl:248
+//line app/vmalert/web.qtpl:231
 				qw422016.E().S(n.Notifier.Addr())
-//line app/vmalert/web.qtpl:248
+//line app/vmalert/web.qtpl:231
 				qw422016.N().S(`</td>
                      </tr>
                  `)
-//line app/vmalert/web.qtpl:250
+//line app/vmalert/web.qtpl:233
 			}
-//line app/vmalert/web.qtpl:250
+//line app/vmalert/web.qtpl:233
 			qw422016.N().S(`
               </tbody>
              </table>
          </div>
      `)
-//line app/vmalert/web.qtpl:254
+//line app/vmalert/web.qtpl:237
 		}
-//line app/vmalert/web.qtpl:254
+//line app/vmalert/web.qtpl:237
 		qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:256
+//line app/vmalert/web.qtpl:239
 	} else {
-//line app/vmalert/web.qtpl:256
+//line app/vmalert/web.qtpl:239
 		qw422016.N().S(`
         <div>
             <p>No items...</p>
         </div>
     `)
-//line app/vmalert/web.qtpl:260
+//line app/vmalert/web.qtpl:243
 	}
-//line app/vmalert/web.qtpl:260
+//line app/vmalert/web.qtpl:243
 	qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:262
+//line app/vmalert/web.qtpl:245
 	tpl.StreamFooter(qw422016)
-//line app/vmalert/web.qtpl:262
+//line app/vmalert/web.qtpl:245
 	qw422016.N().S(`
 
 `)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 }
 
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 func WriteListTargets(qq422016 qtio422016.Writer, targets map[notifier.TargetType][]notifier.Target) {
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	StreamListTargets(qw422016, targets)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 }
 
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 func ListTargets(targets map[notifier.TargetType][]notifier.Target) string {
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	WriteListTargets(qb422016, targets)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 	return qs422016
-//line app/vmalert/web.qtpl:264
+//line app/vmalert/web.qtpl:247
 }
 
-//line app/vmalert/web.qtpl:266
+//line app/vmalert/web.qtpl:249
 func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert) {
-//line app/vmalert/web.qtpl:266
+//line app/vmalert/web.qtpl:249
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:267
+//line app/vmalert/web.qtpl:250
 	tpl.StreamHeader(qw422016, "", navItems)
-//line app/vmalert/web.qtpl:267
+//line app/vmalert/web.qtpl:250
 	qw422016.N().S(`
     `)
-//line app/vmalert/web.qtpl:269
+//line app/vmalert/web.qtpl:252
 	var labelKeys []string
 	for k := range alert.Labels {
 		labelKeys = append(labelKeys, k)
@@ -922,28 +865,28 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
 	}
 	sort.Strings(annotationKeys)
 
-//line app/vmalert/web.qtpl:280
+//line app/vmalert/web.qtpl:263
 	qw422016.N().S(`
     <div class="display-6 pb-3 mb-3">`)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	qw422016.E().S(alert.Name)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	qw422016.N().S(`<span class="ms-2 badge `)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	if alert.State == "firing" {
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 		qw422016.N().S(`bg-danger`)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	} else {
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 		qw422016.N().S(` bg-warning text-dark`)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	}
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	qw422016.E().S(alert.State)
-//line app/vmalert/web.qtpl:281
+//line app/vmalert/web.qtpl:264
 	qw422016.N().S(`</span></div>
     <div class="container border-bottom p-2">
       <div class="row">
@@ -952,9 +895,9 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
           `)
-//line app/vmalert/web.qtpl:288
+//line app/vmalert/web.qtpl:271
 	qw422016.E().S(alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00"))
-//line app/vmalert/web.qtpl:288
+//line app/vmalert/web.qtpl:271
 	qw422016.N().S(`
         </div>
       </div>
@@ -966,9 +909,9 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
           <code><pre>`)
-//line app/vmalert/web.qtpl:298
+//line app/vmalert/web.qtpl:281
 	qw422016.E().S(alert.Expression)
-//line app/vmalert/web.qtpl:298
+//line app/vmalert/web.qtpl:281
 	qw422016.N().S(`</pre></code>
         </div>
       </div>
@@ -980,23 +923,23 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
            `)
-//line app/vmalert/web.qtpl:308
+//line app/vmalert/web.qtpl:291
 	for _, k := range labelKeys {
-//line app/vmalert/web.qtpl:308
+//line app/vmalert/web.qtpl:291
 		qw422016.N().S(`
                 <span class="m-1 badge bg-primary">`)
-//line app/vmalert/web.qtpl:309
+//line app/vmalert/web.qtpl:292
 		qw422016.E().S(k)
-//line app/vmalert/web.qtpl:309
+//line app/vmalert/web.qtpl:292
 		qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:309
+//line app/vmalert/web.qtpl:292
 		qw422016.E().S(alert.Labels[k])
-//line app/vmalert/web.qtpl:309
+//line app/vmalert/web.qtpl:292
 		qw422016.N().S(`</span>
           `)
-//line app/vmalert/web.qtpl:310
+//line app/vmalert/web.qtpl:293
 	}
-//line app/vmalert/web.qtpl:310
+//line app/vmalert/web.qtpl:293
 	qw422016.N().S(`
         </div>
       </div>
@@ -1008,24 +951,24 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
            `)
-//line app/vmalert/web.qtpl:320
+//line app/vmalert/web.qtpl:303
 	for _, k := range annotationKeys {
-//line app/vmalert/web.qtpl:320
+//line app/vmalert/web.qtpl:303
 		qw422016.N().S(`
                 <b>`)
-//line app/vmalert/web.qtpl:321
+//line app/vmalert/web.qtpl:304
 		qw422016.E().S(k)
-//line app/vmalert/web.qtpl:321
+//line app/vmalert/web.qtpl:304
 		qw422016.N().S(`:</b><br>
                 <p>`)
-//line app/vmalert/web.qtpl:322
+//line app/vmalert/web.qtpl:305
 		qw422016.E().S(alert.Annotations[k])
-//line app/vmalert/web.qtpl:322
+//line app/vmalert/web.qtpl:305
 		qw422016.N().S(`</p>
           `)
-//line app/vmalert/web.qtpl:323
+//line app/vmalert/web.qtpl:306
 	}
-//line app/vmalert/web.qtpl:323
+//line app/vmalert/web.qtpl:306
 	qw422016.N().S(`
         </div>
       </div>
@@ -1037,17 +980,17 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
            <a target="_blank" href="`)
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.E().S(path.Join(pathPrefix, "groups"))
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.N().S(`#group-`)
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.E().S(alert.GroupID)
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.E().S(alert.GroupID)
-//line app/vmalert/web.qtpl:333
+//line app/vmalert/web.qtpl:316
 	qw422016.N().S(`</a>
         </div>
       </div>
@@ -1059,132 +1002,132 @@ func StreamAlert(qw422016 *qt422016.Writer, pathPrefix string, alert *APIAlert)
         </div>
         <div class="col">
            <a target="_blank" href="`)
-//line app/vmalert/web.qtpl:343
+//line app/vmalert/web.qtpl:326
 	qw422016.E().S(alert.SourceLink)
-//line app/vmalert/web.qtpl:343
+//line app/vmalert/web.qtpl:326
 	qw422016.N().S(`">Link</a>
         </div>
       </div>
     </div>
     `)
-//line app/vmalert/web.qtpl:347
+//line app/vmalert/web.qtpl:330
 	tpl.StreamFooter(qw422016)
-//line app/vmalert/web.qtpl:347
+//line app/vmalert/web.qtpl:330
 	qw422016.N().S(`
 
 `)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 }
 
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 func WriteAlert(qq422016 qtio422016.Writer, pathPrefix string, alert *APIAlert) {
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	StreamAlert(qw422016, pathPrefix, alert)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 }
 
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 func Alert(pathPrefix string, alert *APIAlert) string {
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	WriteAlert(qb422016, pathPrefix, alert)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 	return qs422016
-//line app/vmalert/web.qtpl:349
+//line app/vmalert/web.qtpl:332
 }
 
-//line app/vmalert/web.qtpl:351
+//line app/vmalert/web.qtpl:334
 func streambadgeState(qw422016 *qt422016.Writer, state string) {
-//line app/vmalert/web.qtpl:351
+//line app/vmalert/web.qtpl:334
 	qw422016.N().S(`
 `)
-//line app/vmalert/web.qtpl:353
+//line app/vmalert/web.qtpl:336
 	badgeClass := "bg-warning text-dark"
 	if state == "firing" {
 		badgeClass = "bg-danger"
 	}
 
-//line app/vmalert/web.qtpl:357
+//line app/vmalert/web.qtpl:340
 	qw422016.N().S(`
 <span class="badge `)
-//line app/vmalert/web.qtpl:358
+//line app/vmalert/web.qtpl:341
 	qw422016.E().S(badgeClass)
-//line app/vmalert/web.qtpl:358
+//line app/vmalert/web.qtpl:341
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:358
+//line app/vmalert/web.qtpl:341
 	qw422016.E().S(state)
-//line app/vmalert/web.qtpl:358
+//line app/vmalert/web.qtpl:341
 	qw422016.N().S(`</span>
 `)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 }
 
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 func writebadgeState(qq422016 qtio422016.Writer, state string) {
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	streambadgeState(qw422016, state)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 }
 
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 func badgeState(state string) string {
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	writebadgeState(qb422016, state)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 	return qs422016
-//line app/vmalert/web.qtpl:359
+//line app/vmalert/web.qtpl:342
 }
 
-//line app/vmalert/web.qtpl:361
+//line app/vmalert/web.qtpl:344
 func streambadgeRestored(qw422016 *qt422016.Writer) {
-//line app/vmalert/web.qtpl:361
+//line app/vmalert/web.qtpl:344
 	qw422016.N().S(`
 <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
 `)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 }
 
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 func writebadgeRestored(qq422016 qtio422016.Writer) {
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	streambadgeRestored(qw422016)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 }
 
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 func badgeRestored() string {
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	writebadgeRestored(qb422016)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 	return qs422016
-//line app/vmalert/web.qtpl:363
+//line app/vmalert/web.qtpl:346
 }
diff --git a/app/vmalert/web_types.go b/app/vmalert/web_types.go
index c49c81b65..fad7be471 100644
--- a/app/vmalert/web_types.go
+++ b/app/vmalert/web_types.go
@@ -4,63 +4,61 @@ import (
 	"time"
 )
 
-// APIAlert represents an notifier.AlertingRule state
+// APIAlert represents a notifier.AlertingRule state
 // for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIAlert struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	RuleID      string            `json:"rule_id"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
 	State       string            `json:"state"`
+	Name        string            `json:"name"`
 	Value       string            `json:"value"`
-	Labels      map[string]string `json:"labels"`
+	Labels      map[string]string `json:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations"`
 	ActiveAt    time.Time         `json:"activeAt"`
-	SourceLink  string            `json:"source"`
-	Restored    bool              `json:"restored"`
+
+	// Additional fields
+
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// RuleID is an unique Rule's ID within a group
+	RuleID string `json:"rule_id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+	// Expression contains the PromQL/MetricsQL expression
+	// for Rule's evaluation
+	Expression string `json:"expression"`
+	// SourceLink contains a link to a system which should show
+	// why Alert was generated
+	SourceLink string `json:"source"`
+	// Restored shows whether Alert's state was restored on restart
+	Restored bool `json:"restored"`
 }
 
 // APIGroup represents Group for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIGroup struct {
-	Name           string             `json:"name"`
-	Type           string             `json:"type"`
-	ID             string             `json:"id"`
-	File           string             `json:"file"`
-	Interval       string             `json:"interval"`
-	Concurrency    int                `json:"concurrency"`
-	Params         []string           `json:"params"`
-	Labels         map[string]string  `json:"labels,omitempty"`
-	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
-	RecordingRules []APIRecordingRule `json:"recording_rules"`
-}
+	// Name is the group name as present in the config
+	Name string `json:"name"`
+	// Rules contains both recording and alerting rules
+	Rules []APIRule `json:"rules"`
+	// Interval is the Group's evaluation interval in float seconds as present in the file.
+	Interval float64 `json:"interval"`
+	// LastEvaluation is the timestamp of the last time the Group was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
 
-// APIAlertingRule represents AlertingRule for WEB view
-type APIAlertingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	For         string            `json:"for"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
-	Annotations map[string]string `json:"annotations"`
-}
+	// Additional fields
 
-// APIRecordingRule represents RecordingRule for WEB view
-type APIRecordingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
+	// Type shows the datasource type (prometheus or graphite) of the Group
+	Type string `json:"type"`
+	// ID is a unique Group ID
+	ID string `json:"id"`
+	// File contains a path to the file with Group's config
+	File string `json:"file"`
+	// Concurrency shows how many rules may be evaluated simultaneously
+	Concurrency int `json:"concurrency"`
+	// Params contains HTTP URL parameters added to each Rule's request
+	Params []string `json:"params,omitempty"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 // GroupAlerts represents a group of alerts for WEB view
@@ -68,3 +66,43 @@ type GroupAlerts struct {
 	Group  APIGroup
 	Alerts []*APIAlert
 }
+
+// APIRule represents a Rule for WEB view
+// see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
+type APIRule struct {
+	// State must be one of these under following scenarios
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state.
+	//  "firing": at least 1 alert in the rule in firing state.
+	//  "inactive": no alert in the rule in firing or pending state.
+	State string `json:"state"`
+	Name  string `json:"name"`
+	// Query represents Rule's `expression` field
+	Query string `json:"query"`
+	// Duration represents Rule's `for` field
+	Duration    float64           `json:"duration"`
+	Labels      map[string]string `json:"labels,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// LastError contains the error faced while executing the rule.
+	LastError string `json:"lastError"`
+	// EvaluationTime is the time taken to completely evaluate the rule in float seconds.
+	EvaluationTime float64 `json:"evaluationTime"`
+	// LastEvaluation is the timestamp of the last time the rule was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
+	// Alerts  is the list of all the alerts in this rule that are currently pending or firing
+	Alerts []*APIAlert `json:"alerts,omitempty"`
+	// Health is the health of rule evaluation.
+	// It MUST be one of "ok", "err", "unknown"
+	Health string `json:"health"`
+	// Type of the rule: recording or alerting
+	Type string `json:"type"`
+
+	// Additional fields
+
+	// Type of the rule: recording or alerting
+	DatasourceType string `json:"datasourceType"`
+	LastSamples    int    `json:"lastSamples"`
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+}
diff --git a/lib/writeconcurrencylimiter/concurrencylimiter.go b/lib/writeconcurrencylimiter/concurrencylimiter.go
index a91fa686d..0237f5ba1 100644
--- a/lib/writeconcurrencylimiter/concurrencylimiter.go
+++ b/lib/writeconcurrencylimiter/concurrencylimiter.go
@@ -31,7 +31,7 @@ func Init() {
 // Do calls f with the limited concurrency.
 func Do(f func() error) error {
 	// Limit the number of conurrent f calls in order to prevent from excess
-	// memory usage and CPU trashing.
+	// memory usage and CPU thrashing.
 	select {
 	case ch <- struct{}{}:
 		err := f()


Test patch:
diff --git a/app/vmalert/web_test.go b/app/vmalert/web_test.go
index e8e820a54..0181988aa 100644
--- a/app/vmalert/web_test.go
+++ b/app/vmalert/web_test.go
@@ -54,9 +54,9 @@ func TestHandler(t *testing.T) {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 	})
-	t.Run("/api/v1/groups", func(t *testing.T) {
+	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/groups", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}