Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1344 (2021-05-30 21:22:03)

new feature: debug relabeling
Use scrape_configs[x].relabel_debug = true to log metric names incl.
labels before and after relabeling. After relabeling related metrics
get dropped, i.e. not submitted to servers.

See #1343


Patch:
diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go
index 048e4b484..ce30eaab9 100644
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -18,6 +18,7 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+	relabelDebug =  flag.Bool("remoteWrite.relabelDebug", false, "Show relabel results of via -remoteWrite.relabelConfig specified rules and skip its submission.")
 )
 
 var labelsGlobal []prompbmarshal.Label
@@ -32,6 +33,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
 		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global.RelabelDebug = *relabelDebug
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
@@ -51,6 +53,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}
+		prc.RelabelDebug = *relabelDebug
 		rcs.perURL[i] = prc
 	}
 	return &rcs, nil
@@ -100,7 +103,11 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 				labels = append(labels, *extraLabel)
 			}
 		}
-		labels = pcs.Apply(labels, labelsLen, true)
+		labels = pcs.Apply(labels, labelsLen, true, pcs.RelabelDebug)
+		if (pcs.RelabelDebug) {
+			// simulate "all labels dropped" to avoid submission
+			labels = labels[:labelsLen]
+		}
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go
index 60e2e7f3b..665f7c9ec 100644
--- a/app/vminsert/relabel/relabel.go
+++ b/app/vminsert/relabel/relabel.go
@@ -101,7 +101,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 	}
 
 	// Apply relabeling
-	tmpLabels = pcs.Apply(tmpLabels, 0, true)
+	tmpLabels = pcs.Apply(tmpLabels, 0, true, pcs.RelabelDebug)
 	ctx.tmpLabels = tmpLabels
 	if len(tmpLabels) == 0 {
 		metricsDropped.Inc()
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 39f144b43..240f03958 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -9,8 +9,11 @@ sort: 15
 * FEATURE: vmalert: add a command-line flag `-rule.configCheckInterval` for automatic re-reading of `-rule` files without the need to send SIGHUP signal. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/512).
 * FEATURE: vmagent: respect the `sample_limit` and `-promscrape.maxScrapeSize` values when scraping targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1331).
 * FEATURE: vmauth: add ability to specify mutliple `url_prefix` entries for balancing the load among multiple `vmselect` and/or `vminsert` nodes in a cluster. See [these docs](https://docs.victoriametrics.com/vmauth.html#load-balancing).
+* FEATURE: vminsert: add `-disableRerouting` command-line flag for forcibly disabling the rerouting. This should help resolving [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/791) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1054) issues.
+* FEATURE: allow building VictoriaMetrics components for Solaris / SmartOS. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1322).
 
 * BUGFIX: reduce CPU usage by up to 2x during querying a database with big number of active daily time series. The issue has been introduced in `v1.59.0`.
+* BUGFIX: vmagent: properly apply auth and tls configs in `eureka_sd_configs`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1350).
 
 
 ## [v1.60.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.60.0)
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 73cb3030c..d5d3bb35f 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -225,6 +225,7 @@ The relabeling can be defined in the following places:
 
 * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels.
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`.
+* By setting the `relabel_debug` property of a `scrape_configs` target section in the `-promscrape.config` file to `true`, one is able to instruct the vmagent to just log the metric before and after relabeling and skip its submission to servers. This way it is much easier to understand and check scraped relabeled metrics before poisoning servers with it.
 * At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
 
diff --git a/go.mod b/go.mod
index e94a56579..9cb2ea939 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/VictoriaMetrics/VictoriaMetrics
 require (
 	cloud.google.com/go v0.82.0 // indirect
 	cloud.google.com/go/storage v1.15.0
-	github.com/VictoriaMetrics/fastcache v1.5.8
+	github.com/VictoriaMetrics/fastcache v1.6.0
 
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
diff --git a/go.sum b/go.sum
index 0ec042d49..2947f6e11 100644
--- a/go.sum
+++ b/go.sum
@@ -96,8 +96,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/VictoriaMetrics/fastcache v1.5.8 h1:XW+YVx9lEXITBVv35ugK9OyotdNJVcbza69o3jmqWuI=
-github.com/VictoriaMetrics/fastcache v1.5.8/go.mod h1:SiMZNgwEPJ9qWLshu9tyuE6bKc9ZWYhcNV/L7jurprQ=
+github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
+github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/VictoriaMetrics/fasthttp v1.0.15 h1:UaX6kOxcQRtwMWBCX5avt2d1IzHp8qK8OUpUswz5akQ=
 github.com/VictoriaMetrics/fasthttp v1.0.15/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKCwY7gRVS3N2JAUJ9jw=
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
@@ -1136,6 +1136,7 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/lib/filestream/filestream_solaris.go b/lib/filestream/filestream_solaris.go
new file mode 100644
index 000000000..631a915d4
--- /dev/null
+++ b/lib/filestream/filestream_solaris.go
@@ -0,0 +1,30 @@
+package filestream
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
+)
+
+func (st *streamTracker) adviseDontNeed(n int, fdatasync bool) error {
+	st.length += uint64(n)
+	if st.fd == 0 {
+		return nil
+	}
+	if st.length < dontNeedBlockSize {
+		return nil
+	}
+	blockSize := st.length - (st.length % dontNeedBlockSize)
+	if fdatasync {
+		if err := unix.Fsync(int(st.fd)); err != nil {
+			return fmt.Errorf("unix.Fsync error: %w", err)
+		}
+	}
+	st.offset += blockSize
+	st.length -= blockSize
+	return nil
+}
+
+func (st *streamTracker) close() error {
+	return nil
+}
diff --git a/lib/fs/fadvise_solaris.go b/lib/fs/fadvise_solaris.go
new file mode 100644
index 000000000..2a158b771
--- /dev/null
+++ b/lib/fs/fadvise_solaris.go
@@ -0,0 +1,8 @@
+package fs
+
+import "os"
+
+func fadviseSequentialRead(f *os.File, prefetch bool) error {
+	// TODO: implement this properly
+	return nil
+}
diff --git a/lib/fs/fs_solaris.go b/lib/fs/fs_solaris.go
new file mode 100644
index 000000000..8cddca829
--- /dev/null
+++ b/lib/fs/fs_solaris.go
@@ -0,0 +1,68 @@
+package fs
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"golang.org/x/sys/unix"
+)
+
+func mmap(fd int, length int) (data []byte, err error) {
+	return unix.Mmap(fd, 0, length, unix.PROT_READ, unix.MAP_SHARED)
+
+}
+func mUnmap(data []byte) error {
+	return unix.Munmap(data)
+}
+
+func mustSyncPath(path string) {
+	d, err := os.Open(path)
+	if err != nil {
+		logger.Panicf("FATAL: cannot open %q: %s", path, err)
+	}
+	if err := d.Sync(); err != nil {
+		_ = d.Close()
+		logger.Panicf("FATAL: cannot flush %q to storage: %s", path, err)
+	}
+	if err := d.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close %q: %s", path, err)
+	}
+}
+
+func createFlockFile(flockFile string) (*os.File, error) {
+	flockF, err := os.Create(flockFile)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create lock file %q: %w", flockFile, err)
+	}
+
+	flock := unix.Flock_t{
+		Type:   unix.F_WRLCK,
+		Start:  0,
+		Len:    0,
+		Whence: 0,
+	}
+	if err := unix.FcntlFlock(flockF.Fd(), unix.F_SETLK, &flock); err != nil {
+		return nil, fmt.Errorf("cannot acquire lock on file %q: %w", flockFile, err)
+	}
+	return flockF, nil
+}
+
+func mustGetFreeSpace(path string) uint64 {
+	d, err := os.Open(path)
+	if err != nil {
+		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
+	}
+	defer MustClose(d)
+
+	fd := d.Fd()
+	var stat unix.Statvfs_t
+	if err := unix.Fstatvfs(int(fd), &stat); err != nil {
+		logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
+	}
+	return freeSpace(stat)
+}
+
+func freeSpace(stat unix.Statvfs_t) uint64 {
+	return uint64(stat.Bavail) * uint64(stat.Bsize)
+}
diff --git a/lib/memory/memory_solaris.go b/lib/memory/memory_solaris.go
new file mode 100644
index 000000000..2e29d9ac6
--- /dev/null
+++ b/lib/memory/memory_solaris.go
@@ -0,0 +1,20 @@
+package memory
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"golang.org/x/sys/unix"
+)
+
+const PHYS_PAGES = 0x1f4
+
+func sysTotalMemory() int {
+	memPageSize := unix.Getpagesize()
+	// https://man7.org/linux/man-pages/man3/sysconf.3.html
+	// _SC_PHYS_PAGES
+	memPagesCnt, err := unix.Sysconf(PHYS_PAGES)
+	if err != nil {
+		logger.Panicf("FATAL: error in unix.Sysconf: %s", err)
+	}
+
+	return memPageSize * int(memPagesCnt)
+}
diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go
index 29d624174..170ad2619 100644
--- a/lib/promrelabel/config.go
+++ b/lib/promrelabel/config.go
@@ -26,6 +26,7 @@ type RelabelConfig struct {
 // ParsedConfigs represents parsed relabel configs.
 type ParsedConfigs struct {
 	prcs []*parsedRelabelConfig
+	RelabelDebug bool
 }
 
 // Len returns the number of relabel configs in pcs.
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index 1ee0e9868..2829efca6 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -35,23 +35,65 @@ func (prc *parsedRelabelConfig) String() string {
 		prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.Regex.String(), prc.Modulus, prc.Replacement, prc.Action)
 }
 
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b []byte
+	b = append(b, '{')
+	mname := ""
+	for _, label := range labels {
+		if (label.Name == "__name__") {
+			mname = label.Value
+			continue
+		}
+		b = append(b, label.Name...)
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		b = append(b, ',')
+	}
+	if b[len(b)-1] == ',' {
+		b[len(b)-1] = '}'
+		return mname + string(b)
+	}
+	return mname
+}
+
 // Apply applies pcs to labels starting from the labelsOffset.
 //
 // If isFinalize is set, then FinalizeLabels is called on the labels[labelsOffset:].
 //
 // The returned labels at labels[labelsOffset:] are sorted.
-func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool) []prompbmarshal.Label {
+func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool, relabelDebug bool) []prompbmarshal.Label {
+	var inStr, outStr string
 	if pcs != nil {
+		if relabelDebug {
+			inStr = labelsToString(labels[labelsOffset:])
+		}
 		for _, prc := range pcs.prcs {
 			tmp := prc.apply(labels, labelsOffset)
 			if len(tmp) == labelsOffset {
 				// All the labels have been removed.
+				if relabelDebug {
+					logger.Infof("\nRelabel  In: %s\nRelabel Out: DROPPED - all labels removed.", inStr)
+				}
 				return tmp
 			}
 			labels = tmp
 		}
 	}
 	labels = removeEmptyLabels(labels, labelsOffset)
+	if relabelDebug {
+		outStr = labelsToString(labels[labelsOffset:])
+		if inStr == outStr {
+			logger.Infof("\nRelabel  In: %s\nRelabel Out: KEPT AS IS - no change.", inStr)
+		} else {
+			logger.Infof("\nRelabel  In: %s\nRelabel Out: %s", inStr, outStr)
+		}
+		// lib/promscrape/config.go::getScrapeWork() would drop the entire
+		// target, if this func does not return a single label. So we can't do
+		// a simple 'labels = labels[:labelsOffset]' here.
+		// BTW: Other callees would just drop the related ts, yes, e.g.
+		// app/vmagent/remotewrite/relabel.go::applyRelabeling() and
+		// lib/promscrape/scrapework.go::addRowToTimeseries()
+	}
 	if isFinalize {
 		labels = FinalizeLabels(labels[:labelsOffset], labels[labelsOffset:])
 	}
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 7ad155915..75d5456ef 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -104,6 +104,7 @@ type ScrapeConfig struct {
 	ProxyURL             proxy.URL                   `yaml:"proxy_url,omitempty"`
 	RelabelConfigs       []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
 	MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
+	RelabelDebug         bool                        `yaml:"relabel_debug,omitempty"`
 	SampleLimit          int                         `yaml:"sample_limit,omitempty"`
 
 	StaticConfigs        []StaticConfig         `yaml:"static_configs,omitempty"`
@@ -581,6 +582,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err)
 	}
+	metricRelabelConfigs.RelabelDebug = sc.RelabelDebug
 	swc := &scrapeWorkConfig{
 		scrapeInterval:       scrapeInterval,
 		scrapeTimeout:        scrapeTimeout,
@@ -820,7 +822,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		// Reduce memory usage by interning all the strings in originalLabels.
 		internLabelStrings(originalLabels)
 	}
-	labels = swc.relabelConfigs.Apply(labels, 0, false)
+	labels = swc.relabelConfigs.Apply(labels, 0, false, swc.metricRelabelConfigs.RelabelDebug)
 	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
 	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
 	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
diff --git a/lib/promscrape/discovery/eureka/eureka.go b/lib/promscrape/discovery/eureka/eureka.go
index 54b7c03f2..1edb980cc 100644
--- a/lib/promscrape/discovery/eureka/eureka.go
+++ b/lib/promscrape/discovery/eureka/eureka.go
@@ -17,7 +17,7 @@ const appsAPIPath = "/apps"
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka
 type SDConfig struct {
 	Server            string                     `yaml:"server,omitempty"`
-	HTTPClientConfig  promauth.HTTPClientConfig  `ymal:",inline"`
+	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
 	ProxyURL          proxy.URL                  `yaml:"proxy_url,omitempty"`
 	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 792234429..5a0ad9cd3 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -512,7 +512,10 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim
 	labelsLen := len(wc.labels)
 	wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels)
 	if needRelabel {
-		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true)
+		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true, sw.Config.MetricRelabelConfigs.RelabelDebug)
+		if sw.Config.MetricRelabelConfigs.RelabelDebug {
+			return	// avoid submission
+		}
 	} else {
 		wc.labels = promrelabel.FinalizeLabels(wc.labels[:labelsLen], wc.labels[labelsLen:])
 		promrelabel.SortLabels(wc.labels[labelsLen:])
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index 31b250f8c..47d29dcbd 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -1898,24 +1898,28 @@ type dateMetricIDCache struct {
 	byDate atomic.Value
 
 	// Contains mutable map protected by mu
-	byDateMutable *byDateMetricIDMap
-	lastSyncTime  uint64
-	mu            sync.Mutex
+	byDateMutable    *byDateMetricIDMap
+	nextSyncDeadline uint64
+	mu               sync.Mutex
 }
 
 func newDateMetricIDCache() *dateMetricIDCache {
 	var dmc dateMetricIDCache
-	dmc.Reset()
+	dmc.resetLocked()
 	return &dmc
 }
 
 func (dmc *dateMetricIDCache) Reset() {
 	dmc.mu.Lock()
+	dmc.resetLocked()
+	dmc.mu.Unlock()
+}
+
+func (dmc *dateMetricIDCache) resetLocked() {
 	// Do not reset syncsCount and resetsCount
 	dmc.byDate.Store(newByDateMetricIDMap())
 	dmc.byDateMutable = newByDateMetricIDMap()
-	dmc.lastSyncTime = fasttime.UnixTimestamp()
-	dmc.mu.Unlock()
+	dmc.nextSyncDeadline = 10 + fasttime.UnixTimestamp()
 
 	atomic.AddUint64(&dmc.resetsCount, 1)
 }
@@ -1948,20 +1952,12 @@ func (dmc *dateMetricIDCache) Has(date, metricID uint64) bool {
 	}
 
 	// Slow path. Check mutable map.
-	currentTime := fasttime.UnixTimestamp()
 	dmc.mu.Lock()
 	v = dmc.byDateMutable.get(date)
 	ok := v.Has(metricID)
-	mustSync := false
-	if currentTime-dmc.lastSyncTime > 10 {
-		mustSync = true
-		dmc.lastSyncTime = currentTime
-	}
+	dmc.syncLockedIfNeeded()
 	dmc.mu.Unlock()
 
-	if mustSync {
-		dmc.sync()
-	}
 	return ok
 }
 
@@ -2000,21 +1996,47 @@ func (dmc *dateMetricIDCache) Set(date, metricID uint64) {
 	dmc.mu.Unlock()
 }
 
-func (dmc *dateMetricIDCache) sync() {
-	dmc.mu.Lock()
+func (dmc *dateMetricIDCache) syncLockedIfNeeded() {
+	currentTime := fasttime.UnixTimestamp()
+	if currentTime >= dmc.nextSyncDeadline {
+		dmc.nextSyncDeadline = currentTime + 10
+		dmc.syncLocked()
+	}
+}
+
+func (dmc *dateMetricIDCache) syncLocked() {
+	if len(dmc.byDateMutable.m) == 0 {
+		// Nothing to sync.
+		return
+	}
 	byDate := dmc.byDate.Load().(*byDateMetricIDMap)
-	for date, e := range dmc.byDateMutable.m {
+	byDateMutable := dmc.byDateMutable
+	for date, e := range byDateMutable.m {
 		v := byDate.get(date)
-		e.v.Union(v)
+		if v == nil {
+			continue
+		}
+		v = v.Clone()
+		v.Union(&e.v)
+		byDateMutable.m[date] = &byDateMetricIDEntry{
+			date: date,
+			v:    *v,
+		}
+	}
+	for date, e := range byDate.m {
+		v := byDateMutable.get(date)
+		if v != nil {
+			continue
+		}
+		byDateMutable.m[date] = e
 	}
 	dmc.byDate.Store(dmc.byDateMutable)
 	dmc.byDateMutable = newByDateMetricIDMap()
-	dmc.mu.Unlock()
 
 	atomic.AddUint64(&dmc.syncsCount, 1)
 
 	if dmc.EntriesCount() > memory.Allowed()/128 {
-		dmc.Reset()
+		dmc.resetLocked()
 	}
 }
 
diff --git a/vendor/github.com/VictoriaMetrics/fastcache/go.mod b/vendor/github.com/VictoriaMetrics/fastcache/go.mod
index f575823ae..783ebfd46 100644
--- a/vendor/github.com/VictoriaMetrics/fastcache/go.mod
+++ b/vendor/github.com/VictoriaMetrics/fastcache/go.mod
@@ -8,4 +8,5 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/snappy v0.0.3
 	github.com/stretchr/testify v1.3.0 // indirect
+	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
 )
diff --git a/vendor/github.com/VictoriaMetrics/fastcache/go.sum b/vendor/github.com/VictoriaMetrics/fastcache/go.sum
index 066369ee9..046d62083 100644
--- a/vendor/github.com/VictoriaMetrics/fastcache/go.sum
+++ b/vendor/github.com/VictoriaMetrics/fastcache/go.sum
@@ -12,3 +12,5 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/vendor/github.com/VictoriaMetrics/fastcache/malloc_mmap.go b/vendor/github.com/VictoriaMetrics/fastcache/malloc_mmap.go
index 424b79b43..e0cd0e761 100644
--- a/vendor/github.com/VictoriaMetrics/fastcache/malloc_mmap.go
+++ b/vendor/github.com/VictoriaMetrics/fastcache/malloc_mmap.go
@@ -5,8 +5,9 @@ package fastcache
 import (
 	"fmt"
 	"sync"
-	"syscall"
 	"unsafe"
+
+	"golang.org/x/sys/unix"
 )
 
 const chunksPerAlloc = 1024
@@ -21,7 +22,7 @@ func getChunk() []byte {
 	if len(freeChunks) == 0 {
 		// Allocate offheap memory, so GOGC won't take into account cache size.
 		// This should reduce free memory waste.
-		data, err := syscall.Mmap(-1, 0, chunkSize*chunksPerAlloc, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
+		data, err := unix.Mmap(-1, 0, chunkSize*chunksPerAlloc, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_ANON|unix.MAP_PRIVATE)
 		if err != nil {
 			panic(fmt.Errorf("cannot allocate %d bytes via mmap: %s", chunkSize*chunksPerAlloc, err))
 		}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index c3c85ea0d..a5a858f9a 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -10,7 +10,7 @@ cloud.google.com/go/internal/version
 # cloud.google.com/go/storage v1.15.0
 ## explicit
 cloud.google.com/go/storage
-# github.com/VictoriaMetrics/fastcache v1.5.8
+# github.com/VictoriaMetrics/fastcache v1.6.0
 ## explicit
 github.com/VictoriaMetrics/fastcache
 # github.com/VictoriaMetrics/fasthttp v1.0.15


Test patch:
diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go
index b0e143b28..975ebe7bc 100644
--- a/lib/storage/storage_test.go
+++ b/lib/storage/storage_test.go
@@ -89,7 +89,9 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 			return fmt.Errorf("c.Has(%d, %d) must return true, but returned false", date, metricID)
 		}
 		if i%11234 == 0 {
-			c.sync()
+			c.mu.Lock()
+			c.syncLocked()
+			c.mu.Unlock()
 		}
 		if i%34323 == 0 {
 			c.Reset()
@@ -103,7 +105,9 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 		metricID := uint64(i) % 123
 		c.Set(date, metricID)
 	}
-	c.sync()
+	c.mu.Lock()
+	c.syncLocked()
+	c.mu.Unlock()
 	for i := 0; i < 1e5; i++ {
 		date := uint64(i) % 3
 		metricID := uint64(i) % 123