Repository: VictoriaMetrics/VictoriaMetrics
Pull request #511 (2020-05-20 21:27:12)

Minor additions to single version Readme


Patch:
diff --git a/README.md b/README.md
index 912b423fb..a0f89bdc1 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,8 @@ in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just downl
 
 Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
+See our [Wiki](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki) for additional documentation.
+
 [Contact us](mailto:info@victoriametrics.com) if you need paid enterprise support for VictoriaMetrics.
 
 
@@ -820,6 +822,8 @@ Data is split in per-month subdirectories inside `<-storageDataPath>/data/small`
 Directories for months outside the configured retention are deleted on the first day of new month.
 In order to keep data according to `-retentionPeriod` max disk space usage is going to be `-retentionPeriod` + 1 month.
 For example if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
+It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod` is set to lower
+value than before then data outside the configured period will be eventually deleted.
 
 ### Multiple retentions
 
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index 1c27f813a..66bd501b5 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -40,6 +40,8 @@ var (
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . See also -promscrape.config.dryRun")
 )
 
 var (
@@ -55,6 +57,21 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+
+	if *dryRun {
+		if err := flag.Set("promscrape.config.strictParse", "true"); err != nil {
+			logger.Panicf("BUG: cannot set promscrape.config.strictParse=true: %s", err)
+		}
+		if err := remotewrite.CheckRelabelConfigs(); err != nil {
+			logger.Fatalf("error when checking relabel configs: %s", err)
+		}
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking Prometheus config: %s", err)
+		}
+		logger.Infof("all the configs are ok; exitting with 0 status code")
+		return
+	}
+
 	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
 	remotewrite.Init()
diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go
index c6390cc81..9f747a7a1 100644
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -32,6 +32,21 @@ var (
 		"Disk usage is unlimited if the value is set to 0")
 )
 
+// CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
+func CheckRelabelConfigs() error {
+	if *relabelConfigPathGlobal != "" {
+		if _, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal); err != nil {
+			return fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %s", *relabelConfigPathGlobal, err)
+		}
+	}
+	for _, path := range *relabelConfigPaths {
+		if _, err := promrelabel.LoadRelabelConfigs(path); err != nil {
+			return fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %s", path, err)
+		}
+	}
+	return nil
+}
+
 var rwctxs []*remoteWriteCtx
 
 // Init initializes remotewrite.
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 1e234d810..8ae3045b7 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -72,6 +72,8 @@ var rollupFuncs = map[string]newRollupFunc{
 	"aggr_over_time":        newRollupFuncTwoArgs(rollupFake),
 	"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
 	"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
+	"ascent_over_time":      newRollupFuncOneArg(rollupAscentOverTime),
+	"descent_over_time":     newRollupFuncOneArg(rollupDescentOverTime),
 
 	// `timestamp` function must return timestamp for the last datapoint on the current window
 	// in order to properly handle offset and timestamps unaligned to the current step.
@@ -116,6 +118,9 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"scrape_interval":     rollupScrapeInterval,
 	"tmin_over_time":      rollupTmin,
 	"tmax_over_time":      rollupTmax,
+	"ascent_over_time":    rollupAscentOverTime,
+	"descent_over_time":   rollupDescentOverTime,
+	"timestamp":           rollupTimestamp,
 }
 
 var rollupFuncsCannotAdjustWindow = map[string]bool{
@@ -138,6 +143,8 @@ var rollupFuncsCannotAdjustWindow = map[string]bool{
 	"increases_over_time": true,
 	"decreases_over_time": true,
 	"integrate":           true,
+	"ascent_over_time":    true,
+	"descent_over_time":   true,
 }
 
 var rollupFuncsRemoveCounterResets = map[string]bool{
@@ -1527,6 +1534,52 @@ func rollupTimestamp(rfa *rollupFuncArg) float64 {
 	return float64(timestamps[len(timestamps)-1]) / 1e3
 }
 
+func rollupAscentOverTime(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	prevValue := rfa.prevValue
+	if math.IsNaN(prevValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		prevValue = values[0]
+		values = values[1:]
+	}
+	s := float64(0)
+	for _, v := range values {
+		d := v - prevValue
+		if d > 0 {
+			s += d
+		}
+		prevValue = v
+	}
+	return s
+}
+
+func rollupDescentOverTime(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	prevValue := rfa.prevValue
+	if math.IsNaN(prevValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		prevValue = values[0]
+		values = values[1:]
+	}
+	s := float64(0)
+	for _, v := range values {
+		d := prevValue - v
+		if d > 0 {
+			s += d
+		}
+		prevValue = v
+	}
+	return s
+}
+
 func rollupFirst(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index 9543f4707..127b75dea 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -120,3 +120,5 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `first_over_time(m[d])` - returns the first value for `m` on the time range `d`.
 - `outliersk(N, m)` - returns up to `N` outlier time series for `m`. Outlier time series have the highest deviation from the `median(m)`.
   This aggregate function is useful to detect anomalies across groups of similar time series.
+- `ascent_over_time(m[d])` - returns the sum of positive deltas between adjancent data points in `m` over `d`. Useful for tracking height gains in GPS track.
+- `descent_over_time(m[d])` - returns the absolute sum of negative deltas between adjancent data points in `m` over `d`. Useful for tracking height loss in GPS track.
diff --git a/docs/vmagent.md b/docs/vmagent.md
index cf3105a2c..ddf7ca715 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -1,8 +1,8 @@
 ## vmagent
 
-`vmagent` is a tiny but brave agent, which helps you collecting metrics from various sources
-and storing them to [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
-or any other Prometheus-compatible storage system that supports `remote_write` protocol.
+`vmagent` is a tiny but brave agent, which helps you collect metrics from various sources
+and stores them in [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports the `remote_write` protocol.
 
 <img alt="vmagent" src="vmagent.png">
 
@@ -11,7 +11,7 @@ or any other Prometheus-compatible storage system that supports `remote_write` p
 
 While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
 and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
-Also, we found that users’ infrastructure is like snowflakes - never alike, and we decided to add more flexibility
+Also, we found that users’ infrastructure are snowflakes - no two are alike, and we decided to add more flexibility
 to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
 
 
@@ -31,7 +31,7 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
 * Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
   to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
-* Uses lower amounts of RAM, CPU, disk IO and network bandwidth comparing to Prometheus.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared to Prometheus.
 
 
 ### Quick Start
@@ -40,8 +40,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. Multiple `-remoteWrite.url` args can be set in parallel
-  in order to replicate data concurrently to multiple remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
 
 Example command line:
 
@@ -49,7 +48,7 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-If you need collecting only Influx data, then the following command line would be enough:
+If you only need to collect Influx data, then the following is sufficient:
 
 ```
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
@@ -79,14 +78,14 @@ See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/Victor
 #### Drop-in replacement for Prometheus
 
 If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
-then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such setup.
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such a setup.
 See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
 
 
 #### Replication and high availability
 
 `vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
-If a single remote storage instance temporarily goes out of service, then the collected data remains available in another remote storage instances.
+If a single remote storage instance temporarily is out of service, then the collected data remains available in another remote storage instances.
 `vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
 Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
 
@@ -94,13 +93,13 @@ Then it sends the buffered data to the remote storage in order to prevent data g
 #### Relabeling and filtering
 
 `vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
-it can remove unneeded samples via Prometheus-like relabeling before sending the collected data to remote storage.
+it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
 See [these docs](#relabeling) for details.
 
 
 #### Splitting data streams among multiple systems
 
-`vmagent` supports splitting of the collected data among muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+`vmagent` supports splitting the collected data between muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
 which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
 data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
 Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
@@ -204,12 +203,12 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
 
-* It is recommended increasing `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+* It is recommended to increase `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
   and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
 
 * `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
-  The directory can grow big when remote storage is unavailable during extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
-  If you don't want sending all the data from the directory to remote storage, just stop `vmagent` and delete the directory.
+  The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
 
 ### How to build from sources
diff --git a/go.mod b/go.mod
index 36db98ea7..04e56aeeb 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.1
 	github.com/VictoriaMetrics/metrics v1.11.3
-	github.com/VictoriaMetrics/metricsql v0.2.2
+	github.com/VictoriaMetrics/metricsql v0.2.3
 	github.com/aws/aws-sdk-go v1.30.28
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/golang/protobuf v1.4.2 // indirect
diff --git a/go.sum b/go.sum
index 284abce9b..e4fc8ad54 100644
--- a/go.sum
+++ b/go.sum
@@ -47,8 +47,8 @@ github.com/VictoriaMetrics/metrics v1.11.2 h1:t/ceLP6SvagUqypCKU7cI7+tQn54+TIV/t
 github.com/VictoriaMetrics/metrics v1.11.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
 github.com/VictoriaMetrics/metrics v1.11.3 h1:eSfXc0CrquKa1VTNUvhP+dhNjLUZHQGTFfp19mYCQWE=
 github.com/VictoriaMetrics/metrics v1.11.3/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
-github.com/VictoriaMetrics/metricsql v0.2.2 h1:3PhBV4g2z7lm8adPShC4vr1PfSkRcLoSq5XOEpSgJPg=
-github.com/VictoriaMetrics/metricsql v0.2.2/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE=
+github.com/VictoriaMetrics/metricsql v0.2.3 h1:xGscDmLoeIV7+8qX/mdHnOY0vu4m+wHIVGMoy/nBovY=
+github.com/VictoriaMetrics/metricsql v0.2.3/go.mod h1:UIjd9S0W1UnTWlJdM0wLS+2pfuPqjwqKoK8yTos+WyE=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/aws/aws-sdk-go v1.30.28 h1:SaPM7dlmp7h3Lj1nJ4jdzOkTdom08+g20k7AU5heZYg=
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 2d07a6918..ed06b938c 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync/atomic"
@@ -25,6 +26,9 @@ import (
 var (
 	strictParse = flag.Bool("promscrape.config.strictParse", false, "Whether to allow only supported fields in '-promscrape.config'. "+
 		"This option may be used for errors detection in '-promscrape.config' file")
+	dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
+		"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
+		"Pass -loggerLevel=ERROR if you don't need to see info messages in the output")
 )
 
 // Config represents essential parts from Prometheus config defined at https://prometheus.io/docs/prometheus/latest/configuration/configuration/
@@ -114,6 +118,13 @@ func loadConfig(path string) (cfg *Config, data []byte, err error) {
 	if err := cfgObj.parse(data, path); err != nil {
 		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %s", path, err)
 	}
+	if *dryRun {
+		// This is a dirty hack for checking Prometheus config only.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/362
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/508 for details.
+		logger.Infof("Success: the config at %q has no errors; exitting with 0 status code", path)
+		os.Exit(0)
+	}
 	return &cfgObj, data, nil
 }
 
@@ -139,7 +150,7 @@ func (cfg *Config) parse(data []byte, path string) error {
 
 func unmarshalMaybeStrict(data []byte, dst interface{}) error {
 	var err error
-	if *strictParse {
+	if *strictParse || *dryRun {
 		err = yaml.UnmarshalStrict(data, dst)
 	} else {
 		err = yaml.Unmarshal(data, dst)
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 54751283a..900313694 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -37,6 +37,15 @@ var (
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config for details")
 )
 
+// CheckConfig checks -promscrape.config for errors and unsupported options.
+func CheckConfig() error {
+	if *promscrapeConfigFile == "" {
+		return fmt.Errorf("missing -promscrape.config option")
+	}
+	_, _, err := loadConfig(*promscrapeConfigFile)
+	return err
+}
+
 // Init initializes Prometheus scraper with config from the `-promscrape.config`.
 //
 // Scraped data is passed to pushData.
diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
index e36083355..da32c308d 100644
--- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
+++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go
@@ -57,6 +57,8 @@ var rollupFuncs = map[string]bool{
 	"aggr_over_time":        true,
 	"hoeffding_bound_upper": true,
 	"hoeffding_bound_lower": true,
+	"ascent_over_time":      true,
+	"descent_over_time":     true,
 
 	// `timestamp` func has been moved here because it must work properly with offsets and samples unaligned to the current step.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details.
diff --git a/vendor/modules.txt b/vendor/modules.txt
index ff5d9f3fd..3c7fb8db9 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -18,7 +18,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.11.3
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.2.2
+# github.com/VictoriaMetrics/metricsql v0.2.3
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/aws/aws-sdk-go v1.30.28


Test patch:
diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go
index 08a1b055d..06b021e83 100644
--- a/app/vmselect/promql/rollup_test.go
+++ b/app/vmselect/promql/rollup_test.go
@@ -389,6 +389,9 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("ideriv", 0)
 	f("decreases_over_time", 5)
 	f("increases_over_time", 5)
+	f("ascent_over_time", 142)
+	f("descent_over_time", 231)
+	f("timestamp", 0.13)
 }
 
 func TestRollupNewRollupFuncError(t *testing.T) {