Repository: VictoriaMetrics/VictoriaMetrics
Pull request #122 (2019-07-25 14:54:40)

RPM packaging
Package as RPM right from the makefile

Uses same .service file as DEB packaging.

kmansoft: CC: @patsevanton 
codecov-io: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=h1) Report
> Merging [#122](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=desc) into [master](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/106ab78a47bfef240e61b34fb88873993af30e57?src=pr&el=desc) will **decrease** coverage by `0.12%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122/graphs/tree.svg?width=650&token=YND8dYVcXn&height=150&src=pr)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #122      +/-   ##
==========================================
- Coverage   71.27%   71.14%   -0.13%     
==========================================
  Files          86       86              
  Lines       14533    14533              
==========================================
- Hits        10358    10340      -18     
- Misses       3269     3282      +13     
- Partials      906      911       +5
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122/diff?src=pr&el=tree#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `94.02% <0%> (-5.98%)` | :arrow_down: |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122/diff?src=pr&el=tree#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `71.67% <0%> (-0.32%)` | :arrow_down: |
| [lib/storage/partition.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvcGFydGl0aW9uLmdv) | `72.05% <0%> (-0.26%)` | :arrow_down: |
| [lib/storage/merge.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvbWVyZ2UuZ28=) | `84.29% <0%> (+1.65%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=footer). Last update [106ab78...4661650](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/122?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/.gitignore b/.gitignore
index 9879e811b..24ef443a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,6 @@
 /vmstorage-data
 /vmselect-cache
 /package/temp-deb-*
+/package/temp-rpm-*
 /package/*.deb
+/package/*.rpm
diff --git a/app/victoria-metrics/Makefile b/app/victoria-metrics/Makefile
index 9cfcb1e65..cf763dee1 100644
--- a/app/victoria-metrics/Makefile
+++ b/app/victoria-metrics/Makefile
@@ -38,17 +38,35 @@ victoria-metrics-pure:
 victoria-metrics-pure-prod:
 	APP_NAME=victoria-metrics APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
 
-### Packaging for Debian - amd64
+### Packaging as DEB - amd64
 victoria-metrics-package-deb: victoria-metrics
-	echo "GO_BUILDINFO:" "$(GO_BUILDINFO)"
-	echo "PWD:" "$(PWD)"
 	./package/package_deb.sh amd64
 
-### Packaging for Debian - arm64
+### Packaging as DEB - arm64
 victoria-metrics-package-deb-arm64: victoria-metrics-arm64
 	./package/package_deb.sh arm64
 
+### Packaging as DEB - all
 victoria-metrics-package-deb-all: \
         victoria-metrics-package-deb \
         victoria-metrics-package-deb-arm64
 
+### Packaging as RPM - amd64
+victoria-metrics-package-rpm: victoria-metrics
+	./package/package_rpm.sh amd64
+
+### Packaging as RPM - arm64
+victoria-metrics-package-rpm-arm64: victoria-metrics-arm64
+	./package/package_rpm.sh arm64
+
+### Packaging as RPM - all
+victoria-metrics-package-rpm-all: \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64
+
+### Packaging as both DEB and RPM - all
+victoria-metrics-package-deb-rpm-all: \
+        victoria-metrics-package-deb \
+        victoria-metrics-package-deb-arm64 \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64
diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go
index 51eaec99a..a2bfdaad0 100644
--- a/app/vmselect/promql/aggr.go
+++ b/app/vmselect/promql/aggr.go
@@ -312,7 +312,11 @@ func aggrFuncCount(tss []*timeseries) []*timeseries {
 			}
 			count++
 		}
-		dst.Values[i] = float64(count)
+		v := float64(count)
+		if count == 0 {
+			v = nan
+		}
+		dst.Values[i] = v
 	}
 	return tss[:1]
 }
diff --git a/app/vmselect/promql/aggr_incremental.go b/app/vmselect/promql/aggr_incremental.go
index 8b0fe4a66..c4651a541 100644
--- a/app/vmselect/promql/aggr_incremental.go
+++ b/app/vmselect/promql/aggr_incremental.go
@@ -348,7 +348,12 @@ func mergeAggrCount(dst, src *incrementalAggrContext) {
 }
 
 func finalizeAggrCount(iac *incrementalAggrContext) {
-	// Nothing to do
+	dstValues := iac.ts.Values
+	for i, v := range dstValues {
+		if v == 0 {
+			dstValues[i] = nan
+		}
+	}
 }
 
 func updateAggrSum2(iac *incrementalAggrContext, values []float64) {
diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go
index 76e8a51a0..04498459d 100644
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -466,31 +466,19 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	preFunc, rcs := getRollupConfigs(name, rf, ec.Start, ec.End, ec.Step, window, sharedTimestamps)
 	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
 	var tssLock sync.Mutex
+	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
 	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
 			var ts timeseries
-			ts.MetricName.CopyFrom(&tsSQ.MetricName)
-			if len(rc.TagValue) > 0 {
-				ts.MetricName.AddTag("rollup", rc.TagValue)
-			}
-			ts.Values = rc.Do(ts.Values[:0], values, timestamps)
-			ts.Timestamps = sharedTimestamps
-			ts.denyReuse = true
-
+			doRollupForTimeseries(rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps, removeMetricGroup)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
 		}
 		return values, timestamps
 	})
-	if !rollupFuncsKeepMetricGroup[name] {
-		tss = copyTimeseriesMetricNames(tss)
-		for _, ts := range tss {
-			ts.MetricName.ResetMetricGroup()
-		}
-	}
 	return tss, nil
 }
 
@@ -614,22 +602,16 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 	defer rml.Put(uint64(rollupMemorySize))
 
 	// Evaluate rollup
+	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
 	var tss []*timeseries
 	if iafc != nil {
-		tss, err = evalRollupWithIncrementalAggregate(iafc, rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupWithIncrementalAggregate(iafc, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
 	} else {
-		tss, err = evalRollupNoIncrementalAggregate(rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupNoIncrementalAggregate(rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
 	}
 	if err != nil {
 		return nil, err
 	}
-
-	if !rollupFuncsKeepMetricGroup[name] {
-		tss = copyTimeseriesMetricNames(tss)
-		for _, ts := range tss {
-			ts.MetricName.ResetMetricGroup()
-		}
-	}
 	tss = mergeTimeseries(tssCached, tss, start, ec)
 	rollupResultCacheV.Put(name, ec, me, iafc, window, tss)
 
@@ -649,21 +631,19 @@ func getRollupMemoryLimiter() *memoryLimiter {
 }
 
 func evalRollupWithIncrementalAggregate(iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
 		defer putTimeseries(ts)
 		for _, rc := range rcs {
 			ts.Reset()
-			ts.MetricName.CopyFrom(&rs.MetricName)
-			if len(rc.TagValue) > 0 {
-				ts.MetricName.AddTag("rollup", rc.TagValue)
-			}
-			ts.Values = rc.Do(ts.Values[:0], rs.Values, rs.Timestamps)
-			ts.Timestamps = sharedTimestamps
+			doRollupForTimeseries(rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
 			iafc.updateTimeseries(ts, workerID)
+
+			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
 			ts.Timestamps = nil
+			ts.denyReuse = false
 		}
 	})
 	if err != nil {
@@ -674,21 +654,14 @@ func evalRollupWithIncrementalAggregate(iafc *incrementalAggrFuncContext, rss *n
 }
 
 func evalRollupNoIncrementalAggregate(rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
 			var ts timeseries
-			ts.MetricName.CopyFrom(&rs.MetricName)
-			if len(rc.TagValue) > 0 {
-				ts.MetricName.AddTag("rollup", rc.TagValue)
-			}
-			ts.Values = rc.Do(ts.Values[:0], rs.Values, rs.Timestamps)
-			ts.Timestamps = sharedTimestamps
-			ts.denyReuse = true
-
+			doRollupForTimeseries(rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
@@ -700,6 +673,20 @@ func evalRollupNoIncrementalAggregate(rss *netstorage.Results, rcs []*rollupConf
 	return tss, nil
 }
 
+func doRollupForTimeseries(rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, valuesSrc []float64, timestampsSrc []int64,
+	sharedTimestamps []int64, removeMetricGroup bool) {
+	tsDst.MetricName.CopyFrom(mnSrc)
+	if len(rc.TagValue) > 0 {
+		tsDst.MetricName.AddTag("rollup", rc.TagValue)
+	}
+	if removeMetricGroup {
+		tsDst.MetricName.ResetMetricGroup()
+	}
+	tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc)
+	tsDst.Timestamps = sharedTimestamps
+	tsDst.denyReuse = true
+}
+
 func getRollupConfigs(name string, rf rollupFunc, start, end, step, window int64, sharedTimestamps []int64) (func(values []float64, timestamps []int64), []*rollupConfig) {
 	preFunc := func(values []float64, timestamps []int64) {}
 	if rollupFuncsRemoveCounterResets[name] {
diff --git a/package/package_rpm.sh b/package/package_rpm.sh
new file mode 100755
index 000000000..89e21536f
--- /dev/null
+++ b/package/package_rpm.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+if ! which rpmbuild 2> /dev/null
+then
+	echo "*** Fatal: please install rpmbuild"
+    exit 1
+fi
+
+ARCH="amd64"
+if [[ $# -ge 1 ]]
+then
+    ARCH="$1"
+fi
+
+# Map to Debian architecture
+if [[ "$ARCH" == "amd64" ]]; then
+    RPM_ARCH=x86_64
+	EXENAME_SRC="victoria-metrics"
+elif [[ "$ARCH" == "arm64" ]]; then
+    RPM_ARCH=aarch64
+    EXENAME_SRC="victoria-metrics-arm64"
+else
+    echo "*** Unknown arch $ARCH"
+    exit 1
+fi
+
+PACKDIR="./package"
+TEMPDIR="${PACKDIR}/temp-rpm-${RPM_ARCH}"
+EXENAME_DST="victoria-metrics"
+
+# Pull in version info
+
+VERSION=`cat ${PACKDIR}/VAR_VERSION | perl -ne 'chomp and print'`
+BUILD=`cat ${PACKDIR}/VAR_BUILD | perl -ne 'chomp and print'`
+
+# Create directories
+
+[[ -d "${TEMPDIR}" ]] && rm -rf "${TEMPDIR}"
+
+mkdir -p "${TEMPDIR}" && echo "*** Created   : ${TEMPDIR}"
+
+echo "*** Version   : ${VERSION}-${BUILD}"
+echo "*** Arch      : ${RPM_ARCH}"
+
+OUT_RPM="victoria-metrics-${VERSION}-${BUILD}.${RPM_ARCH}.rpm"
+
+echo "*** Out .rpm  : ${OUT_RPM}"
+
+cat > "${TEMPDIR}/victoria-metrics.spec" <<EOF
+Summary: The best long-term remote storage for Prometheus
+Name: victoria-metrics
+Version: ${VERSION}
+Release: ${BUILD}
+License: Apache License 2.0
+URL: https://victoriametrics.com/
+Group: System
+Packager: Aliaksandr Valialkin
+Requires: libpthread
+Requires: libc
+
+%description
+VictoriaMetrics is fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
+
+%files
+%attr(0744, root, root) /usr/sbin/*
+%attr(0644, root, root) /lib/systemd/system/*
+
+%prep
+mkdir -p \$RPM_BUILD_ROOT/usr/sbin/
+mkdir -p \$RPM_BUILD_ROOT/lib/systemd/system/
+
+cp ${PWD}/bin/${EXENAME_SRC} \$RPM_BUILD_ROOT/usr/sbin/
+cp ${PWD}/package/victoria-metrics.service \$RPM_BUILD_ROOT/lib/systemd/system/
+
+%post
+/usr/bin/systemctl daemon-reload
+
+%preun
+/usr/bin/systemctl stop victoria-metrics
+
+%postun
+/usr/bin/systemctl daemon-reload
+EOF
+
+rpmbuild -bb --target "${RPM_ARCH}" \
+	"${TEMPDIR}/victoria-metrics.spec"
+
+cp "${HOME}/rpmbuild/RPMS/${RPM_ARCH}/${OUT_RPM}" "${PACKDIR}"
+
+echo "*** Created   : ${PACKDIR}/${OUT_RPM}"


Test patch:
diff --git a/app/vmselect/promql/aggr_incremental_test.go b/app/vmselect/promql/aggr_incremental_test.go
index 7e779a124..f9a78b136 100644
--- a/app/vmselect/promql/aggr_incremental_test.go
+++ b/app/vmselect/promql/aggr_incremental_test.go
@@ -81,7 +81,7 @@ func TestIncrementalAggr(t *testing.T) {
 	})
 	t.Run("count", func(t *testing.T) {
 		t.Parallel()
-		valuesExpected := []float64{6, 0, 5, 5}
+		valuesExpected := []float64{6, nan, 5, 5}
 		f("count", valuesExpected)
 	})
 	t.Run("sum2", func(t *testing.T) {
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 7b950e6f4..c22b8fa41 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -2343,10 +2343,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`count(multi-vector)`, func(t *testing.T) {
 		t.Parallel()
-		q := `count(label_set(10, "foo", "bar") or label_set((15-time()/100)^0.5, "baz", "sss"))`
+		q := `count(label_set(time()<1500, "foo", "bar") or label_set(time()<1800, "baz", "sss"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{2, 2, 2, 1, 1, 1},
+			Values:     []float64{2, 2, 2, 1, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}