Repository: VictoriaMetrics/VictoriaMetrics
Pull request #652 (2020-07-25 16:28:55)

app/vmalert: support `external.label` to specify global labelset for all rules #622
`external.label` flag supposed to help to distinguish alert or recording rules
source in situations when more than one `vmalert` runs for the same datasource
or AlertManager.

codecov[bot]: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=h1) Report
> Merging [#652](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=desc) into [master](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/b9c8f6bf34d81e398c9f46a02eea71081d574c98&el=desc) will **decrease** coverage by `0.00%`.
> The diff coverage is `28.57%`.

[![Impacted file tree graph](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/graphs/tree.svg?width=650&height=150&src=pr&token=YND8dYVcXn)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #652      +/-   ##
==========================================
- Coverage   68.08%   68.08%   -0.01%     
==========================================
  Files         158      158              
  Lines       21646    21666      +20     
==========================================
+ Hits        14738    14751      +13     
- Misses       5708     5713       +5     
- Partials     1200     1202       +2     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [app/vmalert/config/config.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvY29uZmlnL2NvbmZpZy5nbw==) | `81.90% <0.00%> (-3.25%)` | :arrow_down: |
| [app/vmalert/main.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvbWFpbi5nbw==) | `24.29% <0.00%> (-1.45%)` | :arrow_down: |
| [app/vmalert/group.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvZ3JvdXAuZ28=) | `73.65% <30.00%> (-1.98%)` | :arrow_down: |
| [app/vmalert/manager.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvbWFuYWdlci5nbw==) | `82.85% <50.00%> (ø)` | |
| [app/vmalert/alerting.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvYWxlcnRpbmcuZ28=) | `82.82% <66.66%> (-0.77%)` | :arrow_down: |
| [lib/storage/index\_db.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvaW5kZXhfZGIuZ28=) | `75.73% <0.00%> (-0.17%)` | :arrow_down: |
| [lib/storage/partition.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvcGFydGl0aW9uLmdv) | `71.54% <0.00%> (ø)` | |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `68.99% <0.00%> (+0.69%)` | :arrow_up: |
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `87.66% <0.00%> (+1.29%)` | :arrow_up: |
| [lib/storage/merge.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvbWVyZ2UuZ28=) | `84.29% <0.00%> (+3.30%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=footer). Last update [b9c8f6b...5d910cc](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/652?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/README.md b/README.md
index cd4a6c348..e8e440875 100644
--- a/README.md
+++ b/README.md
@@ -394,6 +394,7 @@ or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/mas
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
 and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
+The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/build/html/PushingData.html).
 
 #### Sending data via `telnet put` protocol
 
diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go
index 7dcadcdfe..77c92ebf6 100644
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -182,6 +182,9 @@ func (c *client) runWorker() {
 }
 
 func (c *client) sendBlock(block []byte) {
+	retryDuration := time.Second
+
+again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
 		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.remoteWriteURL, err)
@@ -195,9 +198,6 @@ func (c *client) sendBlock(block []byte) {
 		req.Header.Set("Authorization", c.authHeader)
 	}
 
-	retryDuration := time.Second
-
-again:
 	startTime := time.Now()
 	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)
diff --git a/app/vmalert/Makefile b/app/vmalert/Makefile
index 3f8ad175b..d1877839a 100644
--- a/app/vmalert/Makefile
+++ b/app/vmalert/Makefile
@@ -64,6 +64,8 @@ run-vmalert: vmalert
 		-notifier.url=http://127.0.0.1:9093 \
 		-remoteWrite.url=http://localhost:8428 \
 		-remoteRead.url=http://localhost:8428 \
+		-external.label=cluster=east-1 \
+		-external.label=replica=a \
 		-evaluationInterval=3s
 
 vmalert-amd64:
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 07aba91ee..88c85d8e0 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -44,10 +44,19 @@ compatible storage address for storing recording rules results and alerts state
 Then configure `vmalert` accordingly:
 ```
 ./bin/vmalert -rule=alert.rules \
-		-datasource.url=http://localhost:8428 \
-        -notifier.url=http://localhost:9093
+    -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
+    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
+    -remoteWrite.url=http://localhost:8428 \ # remote write compatible storage to persist rules
+    -remoteRead.url=http://localhost:8428 \  # PromQL compatible datasource to restore alerts state from
+    -external.label=cluster=east-1 \         # External label to be applied for each rule
+    -external.label=replica=a \              # Multiple external labels may be set
+    -evaluationInterval=3s                   # Default evaluation interval if not specified in rules group
 ```
 
+If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
+to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts. 
+
 Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) 
 and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very 
 similar to Prometheus rules and configured using YAML. Configuration examples may be found 
@@ -174,6 +183,8 @@ Usage of vmalert:
         How often to evaluate the rules (default 1m0s)
   -external.url string
         External URL is used as alert's source for sent alerts to the notifier
+  -external.label array
+        Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
   -httpListenAddr string
         Address to listen for http connections (default ":8880")
   -metricsAuthKey string
diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go
index 42fa5e39c..eda3d0550 100644
--- a/app/vmalert/alerting.go
+++ b/app/vmalert/alerting.go
@@ -331,15 +331,22 @@ func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) p
 // Restore restores only Start field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
-func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
 	if q == nil {
 		return fmt.Errorf("querier is nil")
 	}
+
+	// account for external labels in filter
+	var labelsFilter string
+	for k, v := range labels {
+		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
+	}
+
 	// Get the last datapoint in range via MetricsQL `last_over_time`.
 	// We don't use plain PromQL since Prometheus doesn't support
 	// remote write protocol which is used for state persistence in vmalert.
-	expr := fmt.Sprintf("last_over_time(%s{alertname=%q}[%ds])",
-		alertForStateMetricName, ar.Name, int(lookback.Seconds()))
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
+		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
 	qMetrics, err := q.Query(ctx, expr)
 	if err != nil {
 		return err
diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go
index 7adcd6289..3d2ce6dfd 100644
--- a/app/vmalert/config/config.go
+++ b/app/vmalert/config/config.go
@@ -90,6 +90,14 @@ func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+// Name returns Rule name according to its type
+func (r *Rule) Name() string {
+	if r.Record != "" {
+		return r.Record
+	}
+	return r.Alert
+}
+
 // HashRule hashes significant Rule fields into
 // unique hash value
 func HashRule(r Rule) uint64 {
diff --git a/app/vmalert/group.go b/app/vmalert/group.go
index cccd0f460..052815350 100644
--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -32,7 +32,7 @@ type Group struct {
 	updateCh chan *Group
 }
 
-func newGroup(cfg config.Group, defaultInterval time.Duration) *Group {
+func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
 		Name:        cfg.Name,
 		File:        cfg.File,
@@ -50,6 +50,17 @@ func newGroup(cfg config.Group, defaultInterval time.Duration) *Group {
 	}
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
+		// override rule labels with external labels
+		for k, v := range labels {
+			if prevV, ok := r.Labels[k]; ok {
+				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+					k, prevV, g.Name, r.Name(), k, v)
+			}
+			if r.Labels == nil {
+				r.Labels = map[string]string{}
+			}
+			r.Labels[k] = v
+		}
 		rules[i] = g.newRule(r)
 	}
 	g.Rules = rules
@@ -74,7 +85,7 @@ func (g *Group) ID() uint64 {
 }
 
 // Restore restores alerts state for group rules
-func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
 	for _, rule := range g.Rules {
 		rr, ok := rule.(*AlertingRule)
 		if !ok {
@@ -83,7 +94,7 @@ func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time
 		if rr.For < 1 {
 			continue
 		}
-		if err := rr.Restore(ctx, q, lookback); err != nil {
+		if err := rr.Restore(ctx, q, lookback, labels); err != nil {
 			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
 		}
 	}
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 768cadbf1..754a35dbe 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -40,6 +40,8 @@ absolute path to all .yaml files in root.`)
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
+	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+
+		"Pass multiple -label flags in order to add multiple label sets.")
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
@@ -125,6 +127,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		groups:    make(map[uint64]*Group),
 		querier:   q,
 		notifiers: nts,
+		labels:    map[string]string{},
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -137,6 +140,14 @@ func newManager(ctx context.Context) (*manager, error) {
 		return nil, fmt.Errorf("failed to init remoteRead: %w", err)
 	}
 	manager.rr = rr
+
+	for _, s := range *externalLabels {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		manager.labels[s[:n]] = s[n+1:]
+	}
 	return manager, nil
 }
 
diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go
index 98b505caa..2c749b830 100644
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -21,7 +21,8 @@ type manager struct {
 	rw *remotewrite.Client
 	rr datasource.Querier
 
-	wg sync.WaitGroup
+	wg     sync.WaitGroup
+	labels map[string]string
 
 	groupsMu sync.RWMutex
 	groups   map[uint64]*Group
@@ -64,7 +65,7 @@ func (m *manager) close() {
 
 func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
 	if restore && m.rr != nil {
-		err := group.Restore(ctx, m.rr, *remoteReadLookBack)
+		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
 		if err != nil {
 			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
 		}
@@ -88,7 +89,7 @@ func (m *manager) update(ctx context.Context, path []string, validateTpl, valida
 
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
-		ng := newGroup(cfg, *evaluationInterval)
+		ng := newGroup(cfg, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
diff --git a/app/vminsert/common/insert_ctx.go b/app/vminsert/common/insert_ctx.go
index 6f01c845b..ce6ff849f 100644
--- a/app/vminsert/common/insert_ctx.go
+++ b/app/vminsert/common/insert_ctx.go
@@ -70,11 +70,6 @@ func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.La
 }
 
 func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float64) error {
-	if len(ctx.metricNamesBuf) > 16*1024*1024 {
-		if err := ctx.FlushBufs(); err != nil {
-			return err
-		}
-	}
 	mrs := ctx.mrs
 	if cap(mrs) > len(mrs) {
 		mrs = mrs[:len(mrs)+1]
@@ -86,6 +81,11 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.MetricNameRaw = metricNameRaw
 	mr.Timestamp = timestamp
 	mr.Value = value
+	if len(ctx.metricNamesBuf) > 16*1024*1024 {
+		if err := ctx.FlushBufs(); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go
index 5a648d9d7..d92064cd3 100644
--- a/app/vmselect/promql/aggr.go
+++ b/app/vmselect/promql/aggr.go
@@ -27,7 +27,7 @@ var aggrFuncs = map[string]aggrFunc{
 	"bottomk":      newAggrFuncTopK(true),
 	"topk":         newAggrFuncTopK(false),
 	"quantile":     aggrFuncQuantile,
-	"group":        aggrFuncGroup,
+	"group":        newAggrFunc(aggrFuncGroup),
 
 	// PromQL extension funcs
 	"median":         aggrFuncMedian,
@@ -140,25 +140,20 @@ func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	return aggrFuncExt(afe, args[0], &afa.ae.Modifier, limit, true)
 }
 
-func aggrFuncGroup(afa *aggrFuncArg) ([]*timeseries, error) {
-	args := afa.args
-	if err := expectTransformArgsNum(args, 1); err != nil {
-		return nil, err
-	}
-	afe := func(tss []*timeseries) []*timeseries {
-		// See https://github.com/prometheus/prometheus/commit/72425d4e3d14d209cc3f3f6e10e3240411303399
-		values := tss[0].Values
-		for j := range values {
-			values[j] = 1
+func aggrFuncGroup(tss []*timeseries) []*timeseries {
+	// See https://github.com/prometheus/prometheus/commit/72425d4e3d14d209cc3f3f6e10e3240411303399
+	dst := tss[0]
+	for i := range dst.Values {
+		v := nan
+		for _, ts := range tss {
+			if math.IsNaN(ts.Values[i]) {
+				continue
+			}
+			v = 1
 		}
-		return tss[:1]
-	}
-	limit := afa.ae.Limit
-	if limit > 1 {
-		// Only a single time series per group must be returned
-		limit = 1
+		dst.Values[i] = v
 	}
-	return aggrFuncExt(afe, args[0], &afa.ae.Modifier, limit, false)
+	return tss[:1]
 }
 
 func aggrFuncSum(tss []*timeseries) []*timeseries {
diff --git a/app/vmselect/promql/aggr_incremental.go b/app/vmselect/promql/aggr_incremental.go
index 1321ca73c..5be010532 100644
--- a/app/vmselect/promql/aggr_incremental.go
+++ b/app/vmselect/promql/aggr_incremental.go
@@ -56,9 +56,9 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{
 		keepOriginal: true,
 	},
 	"group": {
-		updateAggrFunc:   updateAggrGroup,
-		mergeAggrFunc:    mergeAggrAny,
-		finalizeAggrFunc: finalizeAggrCommon,
+		updateAggrFunc:   updateAggrCount,
+		mergeAggrFunc:    mergeAggrCount,
+		finalizeAggrFunc: finalizeAggrGroup,
 	},
 }
 
@@ -391,6 +391,17 @@ func finalizeAggrCount(iac *incrementalAggrContext) {
 	}
 }
 
+func finalizeAggrGroup(iac *incrementalAggrContext) {
+	dstValues := iac.ts.Values
+	for i, v := range dstValues {
+		if v == 0 {
+			dstValues[i] = nan
+		} else {
+			dstValues[i] = 1
+		}
+	}
+}
+
 func updateAggrSum2(iac *incrementalAggrContext, values []float64) {
 	dstValues := iac.ts.Values
 	dstCounts := iac.values
@@ -505,17 +516,3 @@ func mergeAggrAny(dst, src *incrementalAggrContext) {
 	dstCounts[0] = srcCounts[0]
 	dst.ts.Values = append(dst.ts.Values[:0], srcValues...)
 }
-
-func updateAggrGroup(iac *incrementalAggrContext, values []float64) {
-	dstCounts := iac.values
-	if dstCounts[0] > 0 {
-		return
-	}
-	for i := range values {
-		dstCounts[i] = 1
-	}
-	for i := range values {
-		values[i] = 1
-	}
-	iac.ts.Values = append(iac.ts.Values[:0], values...)
-}
diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go
index 003074fba..7fdc3ded7 100644
--- a/app/vmselect/promql/transform.go
+++ b/app/vmselect/promql/transform.go
@@ -363,8 +363,9 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 					xxMinIdx = i
 				}
 			}
-			if xxMinIdx+1 == len(leGroup) {
-				// Merge the `inf` bucket with the previous bucket in order to save accuracy.
+			// Merge the leGroup[xxMinIdx] bucket with the smallest adjacent bucket in order to preserve
+			// the maximum accuracy.
+			if xxMinIdx+1 == len(leGroup) || (xxMinIdx > 0 && leGroup[xxMinIdx-1].hits < leGroup[xxMinIdx+1].hits) {
 				xxMinIdx--
 			}
 			leGroup[xxMinIdx+1].hits += leGroup[xxMinIdx].hits
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index 7874135d3..04e86e9cd 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -24,6 +24,7 @@ Feel free [filing a feature request](https://github.com/VictoriaMetrics/Victoria
 This functionality can be tried at [an editable Grafana dashboard](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo).
 
 - [`WITH` templates](https://play.victoriametrics.com/promql/expand-with-exprs). This feature simplifies writing and managing complex queries. Go to [`WITH` templates playground](https://victoriametrics.com/promql/expand-with-exprs) and try it.
+- Range duration in functions such as [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate()) may be omitted. VictoriaMetrics automatically selects range duration depending on the current step used for building the graph. For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
 - All the aggregate functions support optional `limit N` suffix in order to limit the number of output series. For example, `sum(x) by (y) limit 10` limits
   the number of output time series after the aggregation to 10. All the other time series are dropped.
 - Metric names and metric labels may contain escaped chars. For instance, `foo\-bar{baz\=aa="b"}` is valid expression. It returns time series with name `foo-bar` containing label `baz=aa` with value `b`. Additionally, `\xXX` escape sequence is supported, where `XX` is hexadecimal representation of escaped char.
@@ -31,14 +32,13 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
   For instance, `rate(metric[10i] offset 5i)` would return per-second rate over a range covering 10 previous steps with the offset of 5 steps.
 - `offset` may be put anywere in the query. For instance, `sum(foo) offset 24h`.
 - `offset` may be negative. For example, `q offset -1h`.
-- `default` binary operator. `q1 default q2` substitutes `NaN` values from `q1` with the corresponding values from `q2`.
+- [Range duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors) and [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be fractional. For instance, `rate(node_network_receive_bytes_total[1.5m] offset 0.5d)`.
+- `default` binary operator. `q1 default q2` fills gaps in `q1` with the corresponding values from `q2`.
 - `histogram_quantile` accepts optional third arg - `boundsLabel`. In this case it returns `lower` and `upper` bounds for the estimated percentile. See [this issue for details](https://github.com/prometheus/prometheus/issues/5706).
-- `if` binary operator. `q1 if q2` removes values from `q1` for `NaN` values from `q2`.
-- `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for non-`NaN` values from `q2`.
+- `if` binary operator. `q1 if q2` removes values from `q1` for missing values from `q2`.
+- `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for existing values from `q2`.
 - Trailing commas on all the lists are allowed - label filters, function args and with expressions. For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`. This simplifies maintenance of multi-line queries.
 - String literals may be concatenated. This is useful with `WITH` templates: `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`.
-- Range duration in functions such as [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate()) may be omitted. VictoriaMetrics automatically selects range duration depending on the current step used for building the graph. For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
-- [Range duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors) and [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be fractional. For instance, `rate(node_network_receive_bytes_total[1.5m] offset 0.5d)`.
 - Comments starting with `#` and ending with newline. For instance, `up # this is a comment for 'up' metric`.
 - Rollup functions - `rollup(m[d])`, `rollup_rate(m[d])`, `rollup_deriv(m[d])`, `rollup_increase(m[d])`, `rollup_delta(m[d])` - return `min`, `max` and `avg`
   values for all the `m` data points over `d` duration.
@@ -77,7 +77,6 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `median(q)` - median aggregate. Shorthand to `quantile(0.5, q)`.
 - `limitk(k, q)` - limits the number of time series returned from `q` to `k`.
 - `any(q) by (x)` - returns any time series from `q` for each group in `x`.
-  Use `limitk(1, q)` if you need retaining all the labels from `q`.
 - `keep_last_value(q)` - fills missing data (gaps) in `q` with the previous non-empty value.
 - `keep_next_value(q)` - fills missing data (gaps) in `q` with the next non-empty value.
 - `interpolate(q)` - fills missing data (gaps) in `q` with linearly interpolated values.
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index cd4a6c348..e8e440875 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -394,6 +394,7 @@ or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/mas
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
 and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
+The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/build/html/PushingData.html).
 
 #### Sending data via `telnet put` protocol
 
diff --git a/go.mod b/go.mod
index c523f489b..7bf5c2881 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.1
-	github.com/VictoriaMetrics/metrics v1.12.0
+	github.com/VictoriaMetrics/metrics v1.12.1
 	github.com/VictoriaMetrics/metricsql v0.2.9
 	github.com/aws/aws-sdk-go v1.33.9
 	github.com/cespare/xxhash/v2 v2.1.1
diff --git a/go.sum b/go.sum
index 9019d63df..b6e99ca90 100644
--- a/go.sum
+++ b/go.sum
@@ -51,8 +51,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.1 h1:I7YdbswTIW63WxoFoUOSNxeOEGB46rdKUL
 github.com/VictoriaMetrics/fasthttp v1.0.1/go.mod h1:BqgsieH90PR7x97c89j+eqZDloKkDhAEQTwhLw6jw/4=
 github.com/VictoriaMetrics/metrics v1.11.2 h1:t/ceLP6SvagUqypCKU7cI7+tQn54+TIV/tGoxihHvx8=
 github.com/VictoriaMetrics/metrics v1.11.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
-github.com/VictoriaMetrics/metrics v1.12.0 h1:BudxtRYSA6j8H9mzjhXNEIsCPIEUPCb76QwFEptQzvQ=
-github.com/VictoriaMetrics/metrics v1.12.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
+github.com/VictoriaMetrics/metrics v1.12.1 h1:mmd6xo5FOmdF7U6X+huFB1O6HvjlkXsmhOIS1hF/A9g=
+github.com/VictoriaMetrics/metrics v1.12.1/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metricsql v0.2.9 h1:RHLEmt4VNZ2RAqZjmXyRtKpCrtSuYUS1+TyOqfXbHWs=
 github.com/VictoriaMetrics/metricsql v0.2.9/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
diff --git a/lib/storage/metaindex_row.go b/lib/storage/metaindex_row.go
index 8dc1dbdbd..ef93d5deb 100644
--- a/lib/storage/metaindex_row.go
+++ b/lib/storage/metaindex_row.go
@@ -16,10 +16,6 @@ type metaindexRow struct {
 	// TSID is the first TSID in the corresponding index block.
 	TSID TSID
 
-	// BlockHeadersCount is the number of block headers
-	// in the given index block.
-	BlockHeadersCount uint32
-
 	// MinTimestamp is the minimum timestamp in the given index block.
 	MinTimestamp int64
 
@@ -29,6 +25,10 @@ type metaindexRow struct {
 	// IndexBlockOffset is the offset of index block.
 	IndexBlockOffset uint64
 
+	// BlockHeadersCount is the number of block headers
+	// in the given index block.
+	BlockHeadersCount uint32
+
 	// IndexBlockSize is the size of compressed index block.
 	IndexBlockSize uint32
 }
diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go
index b11e1b06d..962ceef5b 100644
--- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go
+++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go
@@ -62,3 +62,8 @@ func WriteProcessMetrics(w io.Writer) {
 	writeGoMetrics(w)
 	writeProcessMetrics(w)
 }
+
+// UnregisterMetric removes metric with the given name from default set.
+func UnregisterMetric(name string) bool {
+	return defaultSet.UnregisterMetric(name)
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 46c888e71..bbe7450b6 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -14,7 +14,7 @@ github.com/VictoriaMetrics/fastcache
 github.com/VictoriaMetrics/fasthttp
 github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
-# github.com/VictoriaMetrics/metrics v1.12.0
+# github.com/VictoriaMetrics/metrics v1.12.1
 github.com/VictoriaMetrics/metrics
 # github.com/VictoriaMetrics/metricsql v0.2.9
 github.com/VictoriaMetrics/metricsql


Test patch:
diff --git a/app/vmalert/alerting_test.go b/app/vmalert/alerting_test.go
index ec9e56f86..cace27e6d 100644
--- a/app/vmalert/alerting_test.go
+++ b/app/vmalert/alerting_test.go
@@ -419,7 +419,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 			fq := &fakeQuerier{}
 			tc.rule.GroupID = fakeGroup.ID()
 			fq.add(tc.metrics...)
-			if err := tc.rule.Restore(context.TODO(), fq, time.Hour); err != nil {
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go
index 5439ebae2..f664868ca 100644
--- a/app/vmalert/group_test.go
+++ b/app/vmalert/group_test.go
@@ -150,7 +150,7 @@ func TestGroupStart(t *testing.T) {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
 	const evalInterval = time.Millisecond
-	g := newGroup(groups[0], evalInterval)
+	g := newGroup(groups[0], evalInterval, map[string]string{"cluster": "east-1"})
 	g.Concurrency = 2
 
 	fn := &fakeNotifier{}
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index ac4d2a1b5..5074b5e07 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -3056,6 +3056,7 @@ func TestExecSuccess(t *testing.T) {
 		t.Parallel()
 		q := `sort(buckets_limit(2, (
 			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
+			alias(label_set(98, "le", "300", "x", "y"), "metric"),
 			alias(label_set(52, "le", "200", "x", "y"), "metric"),
 			alias(label_set(50, "le", "120", "x", "y"), "metric"),
 			alias(label_set(20, "le", "70", "x", "y"), "metric"),
@@ -3064,14 +3065,14 @@ func TestExecSuccess(t *testing.T) {
 		)))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{50, 50, 50, 50, 50, 50},
+			Values:     []float64{52, 52, 52, 52, 52, 52},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.MetricGroup = []byte("metric")
 		r1.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("le"),
-				Value: []byte("120"),
+				Value: []byte("200"),
 			},
 			{
 				Key:   []byte("x"),
diff --git a/lib/protoparser/prometheus/parser_test.go b/lib/protoparser/prometheus/parser_test.go
index d86a19fb4..6a2e0740f 100644
--- a/lib/protoparser/prometheus/parser_test.go
+++ b/lib/protoparser/prometheus/parser_test.go
@@ -180,6 +180,20 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			Timestamp: 789,
 		}},
 	})
+	f(`#                                    _                                            _
+#   ___ __ _ ___ ___  __ _ _ __   __| |_ __ __ _        _____  ___ __   ___  _ __| |_ ___ _ __
+`+"#  / __/ _` / __/ __|/ _` | '_ \\ / _` | '__/ _` |_____ / _ \\ \\/ / '_ \\ / _ \\| '__| __/ _ \\ '__|\n"+`
+# | (_| (_| \__ \__ \ (_| | | | | (_| | | | (_| |_____|  __/>  <| |_) | (_) | |  | ||  __/ |
+#  \___\__,_|___/___/\__,_|_| |_|\__,_|_|  \__,_|      \___/_/\_\ .__/ \___/|_|   \__\___|_|
+#                                                               |_|
+#
+# TYPE cassandra_token_ownership_ratio gauge
+cassandra_token_ownership_ratio 78.9`, &Rows{
+		Rows: []Row{{
+			Metric: "cassandra_token_ownership_ratio",
+			Value:  78.9,
+		}},
+	})
 
 	// Timestamp bigger than 1<<31
 	f("aaa 1123 429496729600", &Rows{