Repository: VictoriaMetrics/VictoriaMetrics
Pull request #814 (2020-10-06 04:09:25)

Fix typos in the vmalert datasource


Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a2e4e11d1..669de3f20 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@master
+        uses: actions/setup-go@main
         with:
           go-version: 1.15
         id: go
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d57b7d466..589084efa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,25 @@
 # tip
 
 
+# [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)
+
+* FEATURE: reduce CPU usage for repeated queries over sliding time window when no new time series are added to the database.
+  Typical use cases: repeated evaluation of alerting rules in [vmalert](https://victoriametrics.github.io/vmalert.html) or dashboard auto-refresh in Grafana.
+* FEATURE: vmagent: add OpenStack service discovery aka [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728 .
+* FEATURE: vmalert: make `-maxIdleConnections` configurable for datasource HTTP client. This option can be used for minimizing connection churn.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/795 .
+* FEATURE: add `-influx.maxLineSize` command-line flag for configuring the maximum size for a single Influx line during parsing.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/807
+
+* BUGFIX: properly handle `inf` values during [background merge of LSM parts](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  Previously `Inf` values could result in `NaN` values for adjancent samples in time series. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/805 .
+* BUGFIX: fill gaps on graphs for `range_*` and `running_*` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/806 .
+* BUGFIX: make a copy of label with new name during relabeling with `action: labelmap` in the same way as Prometheus does.
+  Previously the original label name has been replaced. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/812 .
+* BUGFIX: support parsing floating-point timestamp like Graphite Carbon does. Such timestmaps are truncated to seconds.
+
+
 # [v1.42.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.42.0)
 
 * FEATURE: use all the available CPU cores when accepting data via a single TCP connection
diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go
index 30d1f637e..130974f12 100644
--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -66,6 +67,10 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+		return true
+	}
 	if vminsert.RequestHandler(w, r) {
 		return true
 	}
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index af1de694d..8caf8fa65 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -138,6 +138,10 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
+		return true
+	}
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go
index 88458b827..71fe7f71d 100644
--- a/app/vmalert/datasource/vm.go
+++ b/app/vmalert/datasource/vm.go
@@ -93,7 +93,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Reponse body %s", resp.StatusCode, req.URL, body)
+		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
 	}
 	r := &response{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
@@ -106,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
 	}
 	if r.Data.ResultType != rtVector {
-		return nil, fmt.Errorf("unknown restul type:%s. Expected vector", r.Data.ResultType)
+		return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
 	}
 	return r.metrics()
 }
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index e2ff268b6..073daf7d9 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -28,6 +28,9 @@ var (
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
+	finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
+		"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
+		"disk IO usage and CPU usage")
 	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
 	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
 
@@ -65,6 +68,7 @@ func InitWithoutMetrics() {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
 
+	storage.SetFinalMergeDelay(*finalMergeDelay)
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
 
diff --git a/docs/Articles.md b/docs/Articles.md
index 5c1d7582a..c8ed6ff33 100644
--- a/docs/Articles.md
+++ b/docs/Articles.md
@@ -26,6 +26,7 @@
 * [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://medium.com/@valyala/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)
 * [How to migrate data from Prometheus to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
 * [Filtering and modifying time series during import to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
+* [Anomaly Detection in VictoriaMetrics](https://medium.com/@VictoriaMetrics/anomaly-detection-in-victoriametrics-9528538786a7)
 
 
 ## Third-party articles and slides
diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md
index c0ebb37e5..dbdc81584 100644
--- a/docs/CaseStudies.md
+++ b/docs/CaseStudies.md
@@ -73,13 +73,14 @@ See [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d
 
 Numbers:
 
-* The number of active time series per VictoriaMetrics instance is 20M.
+* The number of active time series per VictoriaMetrics instance is 40M.
 * The total number of time series per VictoriaMetrics instance is 400M+.
-* Ingestion rate per VictoriaMetrics instance is 800K data points per second.
+* Ingestion rate per VictoriaMetrics instance is 1M data points per second.
+* The total number of datapoints per VictoriaMetrics instance is 8 trillions.
 * The average time series churn rate is ~3M per day.
-* The average query rate is ~1K per minute (mostly alert queries).
-* Query duration: median is ~70ms, 99th percentile is ~2sec.
-* Retention: 6 months.
+* The average query rate is ~100 per second (mostly alert queries).
+* Query duration: median is ~70ms, 99th percentile is ~1.5sec.
+* Retention: 3 months.
 
 > Alternatives that we’ve played with before choosing VictoriaMetrics are: federated Prometheus, Cortex, IronDB and Thanos.
 > Points that were critical to us when we were choosing a central tsdb, in order of importance:
diff --git a/lib/promscrape/discovery/openstack/api.go b/lib/promscrape/discovery/openstack/api.go
index 5f456687a..21e0e0745 100644
--- a/lib/promscrape/discovery/openstack/api.go
+++ b/lib/promscrape/discovery/openstack/api.go
@@ -54,7 +54,7 @@ func (cfg *apiConfig) getFreshAPICredentials() (*apiCredentials, error) {
 		return nil, fmt.Errorf("cannot refresh OpenStack api token: %w", err)
 	}
 	cfg.creds = newCreds
-	logger.Infof("successfully refreshed OpenStack api token; next expiration: %s", newCreds.expiration)
+	logger.Infof("successfully refreshed OpenStack api token; expiration: %.3f seconds", time.Until(newCreds.expiration).Seconds())
 	return newCreds, nil
 }
 
diff --git a/lib/protoparser/graphite/parser.go b/lib/protoparser/graphite/parser.go
index 1ba53b474..e679f9524 100644
--- a/lib/protoparser/graphite/parser.go
+++ b/lib/protoparser/graphite/parser.go
@@ -98,12 +98,12 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 	if err != nil {
 		return tagsPool, fmt.Errorf("cannot unmarshal value from %q: %w", tail[:n], err)
 	}
-	ts, err := fastfloat.ParseInt64(tail[n+1:])
+	ts, err := fastfloat.Parse(tail[n+1:])
 	if err != nil {
 		return tagsPool, fmt.Errorf("cannot unmarshal timestamp from %q: %w", tail[n+1:], err)
 	}
 	r.Value = v
-	r.Timestamp = ts
+	r.Timestamp = int64(ts)
 	return tagsPool, nil
 }
 
diff --git a/lib/storage/partition.go b/lib/storage/partition.go
index cefa70c70..f159c0b98 100644
--- a/lib/storage/partition.go
+++ b/lib/storage/partition.go
@@ -958,7 +958,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
 		if !errors.Is(err, errNothingToMerge) {
 			return err
 		}
-		if fasttime.UnixTimestamp()-lastMergeTime > 30 {
+		if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
 			// We have free time for merging into bigger parts.
 			// This should improve select performance.
 			lastMergeTime = fasttime.UnixTimestamp()
@@ -980,6 +980,19 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
 	}
 }
 
+var finalMergeDelaySeconds = uint64(30)
+
+// SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data.
+//
+// This function may be called only before Storage initialization.
+func SetFinalMergeDelay(delay time.Duration) {
+	delaySeconds := int(delay.Seconds() + 0.5)
+	if delaySeconds <= 0 {
+		return
+	}
+	finalMergeDelaySeconds = uint64(delaySeconds)
+}
+
 func maxRowsByPath(path string) uint64 {
 	freeSpace := fs.MustGetFreeSpace(path)
 


Test patch:
diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go
index d7998fb4e..4c59129ae 100644
--- a/app/vmalert/datasource/vm_test.go
+++ b/app/vmalert/datasource/vm_test.go
@@ -75,7 +75,7 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("expected error status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
-		t.Fatalf("expected unkown status got nil")
+		t.Fatalf("expected unknown status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
diff --git a/lib/protoparser/graphite/parser_test.go b/lib/protoparser/graphite/parser_test.go
index 86510f213..6217adfc3 100644
--- a/lib/protoparser/graphite/parser_test.go
+++ b/lib/protoparser/graphite/parser_test.go
@@ -113,6 +113,16 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		}},
 	})
 
+	// Floating-point timestamp
+	// See https://github.com/graphite-project/carbon/blob/b0ba62a62d40a37950fed47a8f6ae6d0f02e6af5/lib/carbon/protocols.py#L197
+	f("aaa 1123 4294.943", &Rows{
+		Rows: []Row{{
+			Metric:    "aaa",
+			Value:     1123,
+			Timestamp: 4294,
+		}},
+	})
+
 	// Tags
 	f("foo;bar=baz 1 2", &Rows{
 		Rows: []Row{{