Repository: VictoriaMetrics/VictoriaMetrics
Pull request #5676 (2024-01-24 11:27:24)

Automatic update operator docs from VictoriaMetrics/operator@f6b9c08


Patch:
diff --git a/Makefile b/Makefile
index 58469993c..e8badd173 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
+MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
diff --git a/app/vmauth/main.go b/app/vmauth/main.go
index c60bb7af9..5774d0e11 100644
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -261,15 +261,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
 		return false
 	}
-	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
-		// Retry requests at other backends if it matches retryStatusCodes.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
-		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
-		return false
+	if hasInt(retryStatusCodes, res.StatusCode) {
+		if rtbOK && rtb.canRetry() {
+			// Retry requests at other backends if it matches retryStatusCodes.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			// NOTE: do not use httpserver.GetRequestURI
+			// it explicitly reads request body, which may fail retries.
+			logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
+				remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
+			return false
+		}
+		// if we get an error from the retry_status_codes list, but cannot execute retry,
+		// we consider such a request an error as well
+		ui.backendErrors.Inc()
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
diff --git a/app/vmselect/main.go b/app/vmselect/main.go
index e9d66d29a..1857f3d80 100644
--- a/app/vmselect/main.go
+++ b/app/vmselect/main.go
@@ -149,8 +149,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
 					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
+				StatusCode: http.StatusTooManyRequests,
 			}
+			w.Header().Add("Retry-After", "10")
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index cca39859c..e9d228625 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -496,11 +496,11 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_composite_filter_success_conversions_total`, idbm.CompositeFilterSuccessConversions)
 	metrics.WriteCounterUint64(w, `vm_composite_filter_missing_conversions_total`, idbm.CompositeFilterMissingConversions)
 
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/inmemory"}`, tm.InmemoryAssistedMerges)
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/small"}`, tm.SmallAssistedMerges)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/inmemory"}`, tm.InmemoryAssistedMergesCount)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/small"}`, tm.SmallAssistedMergesCount)
 
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryAssistedMerges)
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/file"}`, idbm.FileAssistedMerges)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryAssistedMergesCount)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/file"}`, idbm.FileAssistedMergesCount)
 
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_total`, idbm.ItemsAdded)
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_size_bytes_total`, idbm.ItemsAddedSizeBytes)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ff1dc6066..c7f074620 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -62,7 +62,6 @@ The sandbox cluster installation is running under the constant load generated by
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `-vmui.defaultTimezone` flag to set a default timezone. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5375) and [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui#timezone-configuration).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): include UTC in the timezone selection dropdown for standardized time referencing. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5375).
 * FEATURE: add [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/grafana-datasource) to docker compose environment. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5363).
-* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): disable compression for RPC responses sent from `vmstorage` to `vmselect`. This improves query performance at the cost of increased network bandwidth usage. The compression can be enabled back by passing `-rpc.disableCompression=false` command-line flag to `vmstorage`. The compression can be enabled back in [multi-level cluster setup](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup) by passing `-clusternative.disableCompression=false` command-line flag to lower-level `vmselect` nodes.
 
 * BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649).
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly return full results when `-search.skipSlowReplicas` command-line flag is passed to `vmselect` and when [vmstorage groups](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmstorage-groups-at-vmselect) are in use. Previously partial results could be returned in this case.
@@ -79,12 +78,14 @@ The sandbox cluster installation is running under the constant load generated by
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): exit if there is config syntax error in [`scrape_config_files`](https://docs.victoriametrics.com/vmagent.html#loading-scrape-configs-from-multiple-files) when `-promscrape.config.strictParse=true`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5508).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly discover targets for `role: endpoints` and `role: endpointslice` in [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). Previously some `endpoints` and `endpointslice` targets could be left undiscovered or some targets could have missing `__meta_*` labels when performing service discovery in busy Kubernetes clusters with large number of pods. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5557).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): respect explicitly set `series_limit: 0` in [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This allows removing [`series_limit` restriction](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter) on a per-`scrape_config` basis when global limit is set via `-promscrape.seriesLimitPerTarget`. Previously, `0` value was ignored in favor of `-promscrape.seriesLimitPerTarget`.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not discover scrape targets for already terminated pods and containers in [`kubernetes_sd_configs`](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). Such pods and containers cannot be scraped and cannot resurrect, so there is no sense in generating scrape targets for them.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a link for the statistic inaccuracy explanation in the cardinality explorer tool. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5460).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the display of autocomplete results and cache the results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5472) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5470).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): send `step` param for instant queries. The change reverts [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896) due to reasons explained in [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896#issuecomment-1896704401).
 * BUGFIX: all: fix potential panic during components shutdown when [metrics push](https://docs.victoriametrics.com/#push-metrics) is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly process queries with too big lookbehind window such as `foo[100y]`. Previously, such queries could return empty responses even if `foo` is present in database. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle possible negative results caused by float operations precision error in rollup functions like rate() or increase(). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571).
+* BUGFIX: `vmselect`: vmsingle/vmselect returns http status 429 (TooManyRequests) instead of 503 (ServiceUnavailable) when max concurrent requests limit is reached.
 
 ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)
 
diff --git a/docs/CHANGELOG_2023.md b/docs/CHANGELOG_2023.md
index fe510c83b..ccdc7e0aa 100644
--- a/docs/CHANGELOG_2023.md
+++ b/docs/CHANGELOG_2023.md
@@ -590,7 +590,7 @@ created by v1.90.0 or newer versions. The solution is to upgrade to v1.90.0 or n
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-kafka.consumer.topic.concurrency` command-line flag. It controls the number of Kafka consumer workers to use by `vmagent`. It should eliminate the need to start multiple `vmagent` instances to improve data transfer rate. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1957).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [Kafka producer and consumer](https://docs.victoriametrics.com/vmagent.html#kafka-integration) on `arm64` machines. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2271).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): delete unused buffered data at `-remoteWrite.tmpDataPath` directory when there is no matching `-remoteWrite.url` to send this data to. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add the ability for hot reloading of [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) configs. See [these docs](https://docs.victoriametrics.com/stream-aggregation.html#configuration-update) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3639).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add the ability for hot reloading of [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) configs. See [these docs](https://docs.victoriametrics.com/stream-aggregation.html#configuration-update) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3969).
 * FEATURE: check the contents of `-relabelConfig` and `-streamAggr.config` files additionally to `-promscrape.config` when single-node VictoriaMetrics runs with `-dryRun` command-line flag. This aligns the behaviour of single-node VictoriaMetrics with [vmagent](https://docs.victoriametrics.com/vmagent.html) behaviour for `-dryRun` command-line flag.
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): automatically draw a heatmap graph when the query selects a single [histogram](https://docs.victoriametrics.com/keyConcepts.html#histogram). This simplifies analyzing histograms. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3384).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add support for drag'n'drop and paste from clipboard in the "Trace analyzer" page. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3971).
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index fe5da5d38..134ff1901 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -596,7 +596,7 @@ Some capacity planning tips for VictoriaMetrics cluster:
 - Query latency can be reduced by increasing CPU resources per each `vmselect` node, since each incoming query is processed by a single `vmselect` node. Performance for heavy queries scales with the number of available CPU cores at `vmselect` node, since `vmselect` processes time series referred by the query on all the available CPU cores.
 - If the cluster needs to process incoming queries at a high rate, then its capacity can be increased by adding more `vmselect` nodes, so incoming queries could be spread among bigger number of `vmselect` nodes.
 - By default `vminsert` compresses the data it sends to `vmstorage` in order to reduce network bandwidth usage. The compression takes additional CPU resources at `vminsert`. If `vminsert` nodes have limited CPU, then the compression can be disabled by passing `-rpc.disableCompression` command-line flag at `vminsert` nodes.
-- By default `vmstorage` doesn't compress the data it sends to `vmselect` in order to reduce CPU usage at the cost of additional network bandwidth usage. Pass `-rpc.disableCompression=false` command-line flag at `vmstorage` in order to reduce network bandwidh usage needed for processing queries at the cost of increased CPU usage.
+- By default `vmstorage` compresses the data it sends to `vmselect` during queries in order to reduce network bandwidth usage. The compression takes additional CPU resources at `vmstorage`. If `vmstorage` nodes have limited CPU, then the compression can be disabled by passing `-rpc.disableCompression` command-line flag at `vmstorage` nodes.
 
 See also [resource usage limits docs](#resource-usage-limits).
 
@@ -1187,7 +1187,7 @@ Below is the output for `/path/to/vmselect -help`:
   -cluster.tlsKeyFile string
      Path to client-side TLS key file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -clusternative.disableCompression
-     Whether to disable compression of the data sent to vmselect via -clusternativeListenAddr. This reduces CPU usage at the cost of higher network bandwidth usage (default true)
+     Whether to disable compression of the data sent to vmselect via -clusternativeListenAddr. This reduces CPU usage at the cost of higher network bandwidth usage
   -clusternative.maxConcurrentRequests int
      The maximum number of concurrent vmselect requests the server can process at -clusternativeListenAddr. It shouldn't be high, since a single request usually saturates a CPU core at the underlying vmstorage nodes, and many concurrently executed requests may require high amounts of memory. See also -clusternative.maxQueueDuration (default 32)
   -clusternative.maxQueueDuration duration
@@ -1608,7 +1608,7 @@ Below is the output for `/path/to/vmstorage -help`:
   -retentionTimezoneOffset duration
      The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
   -rpc.disableCompression
-     Whether to disable compression of the data sent from vmstorage to vmselect. This reduces CPU usage at the cost of higher network bandwidth usage (default true)
+     Whether to disable compression of the data sent from vmstorage to vmselect. This reduces CPU usage at the cost of higher network bandwidth usage
   -search.maxConcurrentRequests int
      The maximum number of concurrent vmselect requests the vmstorage can process at -vmselectAddr. It shouldn't be high, since a single request usually saturates a CPU core, and many concurrently executed requests may require high amounts of memory. See also -search.maxQueueDuration (default 32)
   -search.maxQueueDuration duration
diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md
index 21e2188fb..ea7c913f6 100644
--- a/docs/operator/CHANGELOG.md
+++ b/docs/operator/CHANGELOG.md
@@ -16,11 +16,14 @@ aliases:
 
 ## Next release
 
+<a name="v0.40.0"></a>
+## [v0.40.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.40.0) - 23 Jan 2024
+
 - [vmalertmanager](./api.html#vmalertmanagerconfig): fix `VMAlertmanagerConfig` discovery according to [the docs](https://docs.victoriametrics.com/operator/resources/vmalertmanager.html#using-vmalertmanagerconfig).
 - [vmoperator](./README.md): add alerting rules for operator itself. See [this issue](https://github.com/VictoriaMetrics/operator/issues/526) for details.
 - [vmoperator](./README.md): add `revisionHistoryLimitCount` field for victoriametrics workload CRDs. See [this issue](https://github.com/VictoriaMetrics/operator/pull/834) for details. Thanks [@gidesh](https://github.com/gidesh)
-- [vmuser](./api.md#vmuser): add new fields to VMUser: `drop_src_path_prefix_parts`, `tls_insecure_skip_verify`, `metric_labels` and `load_balancing_policy`. See [specifications](https://docs.victoriametrics.com/operator/api.html#vmuserspec) and [vmauth docs](https://docs.victoriametrics.com/vmauth.htm) for more details.
-- [vmoperator](./README.md): add CRD support for discord_configs, msteams_configs, sns_configs and webex_configs receiver types in VMAlertmanagerConfig. See [this issue](https://github.com/VictoriaMetrics/operator/issues/808)
+- [vmuser](./api.md#vmuser): add new fields to VMUser: `drop_src_path_prefix_parts`, `tls_insecure_skip_verify`, `metric_labels` and `load_balancing_policy`. See [specifications](https://docs.victoriametrics.com/operator/api.html#vmuserspec) and [vmauth docs](https://docs.victoriametrics.com/vmauth.htm) for more details. **Field `metric_labels` will work only with VMAuth version >= v1.97.0!**
+- [vmoperator](./README.md): add CRD support for `discord_configs`, `msteams_configs`, `sns_configs` and `webex_configs` receiver types in [VMAlertmanagerConfig](https://docs.victoriametrics.com/operator/resources/vmalertmanagerconfig.html). See [this issue](https://github.com/VictoriaMetrics/operator/issues/808)
 
 <a name="v0.39.4"></a>
 ## [v0.39.4](https://github.com/VictoriaMetrics/operator/releases/tag/v0.39.4) - 13 Dec 2023
diff --git a/docs/operator/vars.md b/docs/operator/vars.md
index 413580070..94280b0f8 100644
--- a/docs/operator/vars.md
+++ b/docs/operator/vars.md
@@ -10,7 +10,7 @@ menu:
 
 <!-- this doc autogenerated - don't edit it manually -->
 # Auto Generated vars for package config 
- updated at Tue Jan 23 15:16:59 UTC 2024 
+ updated at Wed Jan 24 11:27:21 UTC 2024 
 
 
 | varible name | variable default value | variable required | variable description |
diff --git a/lib/mergeset/table.go b/lib/mergeset/table.go
index b0680ed9f..b4e64dbe4 100644
--- a/lib/mergeset/table.go
+++ b/lib/mergeset/table.go
@@ -110,8 +110,8 @@ type Table struct {
 	inmemoryItemsMerged uint64
 	fileItemsMerged     uint64
 
-	inmemoryAssistedMerges uint64
-	fileAssistedMerges     uint64
+	inmemoryAssistedMergesCount uint64
+	fileAssistedMergesCount     uint64
 
 	itemsAdded          uint64
 	itemsAddedSizeBytes uint64
@@ -422,8 +422,8 @@ type TableMetrics struct {
 	InmemoryItemsMerged uint64
 	FileItemsMerged     uint64
 
-	InmemoryAssistedMerges uint64
-	FileAssistedMerges     uint64
+	InmemoryAssistedMergesCount uint64
+	FileAssistedMergesCount     uint64
 
 	ItemsAdded          uint64
 	ItemsAddedSizeBytes uint64
@@ -473,8 +473,8 @@ func (tb *Table) UpdateMetrics(m *TableMetrics) {
 	m.InmemoryItemsMerged += atomic.LoadUint64(&tb.inmemoryItemsMerged)
 	m.FileItemsMerged += atomic.LoadUint64(&tb.fileItemsMerged)
 
-	m.InmemoryAssistedMerges += atomic.LoadUint64(&tb.inmemoryAssistedMerges)
-	m.FileAssistedMerges += atomic.LoadUint64(&tb.fileAssistedMerges)
+	m.InmemoryAssistedMergesCount += atomic.LoadUint64(&tb.inmemoryAssistedMergesCount)
+	m.FileAssistedMergesCount += atomic.LoadUint64(&tb.fileAssistedMergesCount)
 
 	m.ItemsAdded += atomic.LoadUint64(&tb.itemsAdded)
 	m.ItemsAddedSizeBytes += atomic.LoadUint64(&tb.itemsAddedSizeBytes)
@@ -792,7 +792,7 @@ func (tb *Table) assistedMergeForInmemoryParts() {
 		return
 	}
 
-	atomic.AddUint64(&tb.inmemoryAssistedMerges, 1)
+	atomic.AddUint64(&tb.inmemoryAssistedMergesCount, 1)
 
 	maxOutBytes := tb.getMaxFilePartSize()
 
@@ -812,13 +812,13 @@ func (tb *Table) assistedMergeForInmemoryParts() {
 
 func (tb *Table) assistedMergeForFileParts() {
 	tb.partsLock.Lock()
-	needMerge := getNotInMergePartsCount(tb.fileParts) >= defaultPartsToMerge
+	needMerge := len(tb.fileParts) > maxFileParts && getNotInMergePartsCount(tb.fileParts) >= defaultPartsToMerge
 	tb.partsLock.Unlock()
 	if !needMerge {
 		return
 	}
 
-	atomic.AddUint64(&tb.fileAssistedMerges, 1)
+	atomic.AddUint64(&tb.fileAssistedMergesCount, 1)
 	err := tb.mergeExistingParts(false)
 	if err == nil {
 		return
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 7ab1a3591..4b6ca982b 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -133,7 +133,7 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 				m := promutils.GetLabels()
 				m.Add("__address__", addr)
 				p.appendCommonLabels(m, gw)
-				p.appendContainerLabels(m, c, &cp)
+				p.appendContainerLabels(m, &c, &cp)
 
 				// Prometheus sets endpoints_name and namespace labels for all endpoints
 				// Even if port is not matching service port.
@@ -189,7 +189,7 @@ func getEndpointLabelsForAddressAndPort(gw *groupWatcher, podPortsSeen map[*Pod]
 		for _, cp := range c.Ports {
 			if cp.ContainerPort == epp.Port {
 				podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
-				p.appendContainerLabels(m, c, &cp)
+				p.appendContainerLabels(m, &c, &cp)
 				break
 			}
 		}
diff --git a/lib/promscrape/discovery/kubernetes/endpointslice.go b/lib/promscrape/discovery/kubernetes/endpointslice.go
index cfd1b5e55..e0e969afe 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslice.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslice.go
@@ -83,7 +83,7 @@ func (eps *EndpointSlice) getTargetLabels(gw *groupWatcher) []*promutils.Labels
 				m := promutils.GetLabels()
 				m.Add("__address__", addr)
 				p.appendCommonLabels(m, gw)
-				p.appendContainerLabels(m, c, &cp)
+				p.appendContainerLabels(m, &c, &cp)
 
 				// Prometheus sets endpoints_name and namespace labels for all endpoints
 				// Even if port is not matching service port.
@@ -127,7 +127,7 @@ func getEndpointSliceLabelsForAddressAndPort(gw *groupWatcher, podPortsSeen map[
 		for _, cp := range c.Ports {
 			if cp.ContainerPort == epp.Port {
 				podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
-				p.appendContainerLabels(m, c, &cp)
+				p.appendContainerLabels(m, &c, &cp)
 				break
 			}
 		}
diff --git a/lib/promscrape/discovery/kubernetes/pod.go b/lib/promscrape/discovery/kubernetes/pod.go
index 5c89a5a59..4ea474e5e 100644
--- a/lib/promscrape/discovery/kubernetes/pod.go
+++ b/lib/promscrape/discovery/kubernetes/pod.go
@@ -104,19 +104,51 @@ type PodCondition struct {
 type ContainerStatus struct {
 	Name        string
 	ContainerID string
+	State       ContainerState
+}
+
+// ContainerState implements k8s container state.
+//
+// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core
+type ContainerState struct {
+	Terminated *ContainerStateTerminated
+}
+
+// ContainerStateTerminated implements k8s terminated container state.
+//
+// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core
+type ContainerStateTerminated struct {
+	ExitCode int
 }
 
 func getContainerID(p *Pod, containerName string, isInit bool) string {
+	cs := p.getContainerStatus(containerName, isInit)
+	if cs == nil {
+		return ""
+	}
+	return cs.ContainerID
+}
+
+func isContainerTerminated(p *Pod, containerName string, isInit bool) bool {
+	cs := p.getContainerStatus(containerName, isInit)
+	if cs == nil {
+		return false
+	}
+	return cs.State.Terminated != nil
+}
+
+func (p *Pod) getContainerStatus(containerName string, isInit bool) *ContainerStatus {
 	css := p.Status.ContainerStatuses
 	if isInit {
 		css = p.Status.InitContainerStatuses
 	}
-	for _, cs := range css {
+	for i := range css {
+		cs := &css[i]
 		if cs.Name == containerName {
-			return cs.ContainerID
+			return cs
 		}
 	}
-	return ""
+	return nil
 }
 
 // getTargetLabels returns labels for each port of the given p.
@@ -124,28 +156,42 @@ func getContainerID(p *Pod, containerName string, isInit bool) string {
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
 func (p *Pod) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 	if len(p.Status.PodIP) == 0 {
-		// Skip pod without IP
+		// Skip pod without IP, since such pods cannnot be scraped.
+		return nil
+	}
+	if isPodPhaseFinished(p.Status.Phase) {
+		// Skip already stopped pod, since it cannot be scraped.
 		return nil
 	}
+
 	var ms []*promutils.Labels
 	ms = appendPodLabels(ms, gw, p, p.Spec.Containers, false)
 	ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, true)
 	return ms
 }
 
+func isPodPhaseFinished(phase string) bool {
+	// See https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+	return phase == "Succeeded" || phase == "Failed"
+
+}
 func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit bool) []*promutils.Labels {
 	for _, c := range cs {
+		if isContainerTerminated(p, c.Name, isInit) {
+			// Skip terminated containers
+			continue
+		}
 		for _, cp := range c.Ports {
-			ms = appendPodLabelsInternal(ms, gw, p, c, &cp, isInit)
+			ms = appendPodLabelsInternal(ms, gw, p, &c, &cp, isInit)
 		}
 		if len(c.Ports) == 0 {
-			ms = appendPodLabelsInternal(ms, gw, p, c, nil, isInit)
+			ms = appendPodLabelsInternal(ms, gw, p, &c, nil, isInit)
 		}
 	}
 	return ms
 }
 
-func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit bool) []*promutils.Labels {
+func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c *Container, cp *ContainerPort, isInit bool) []*promutils.Labels {
 	addr := p.Status.PodIP
 	if cp != nil {
 		addr = discoveryutils.JoinHostPort(addr, cp.ContainerPort)
@@ -168,7 +214,7 @@ func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c
 	return append(ms, m)
 }
 
-func (p *Pod) appendContainerLabels(m *promutils.Labels, c Container, cp *ContainerPort) {
+func (p *Pod) appendContainerLabels(m *promutils.Labels, c *Container, cp *ContainerPort) {
 	m.Add("__meta_kubernetes_pod_container_image", c.Image)
 	m.Add("__meta_kubernetes_pod_container_name", c.Name)
 	if cp != nil {
diff --git a/lib/storage/partition.go b/lib/storage/partition.go
index 8861a0afa..66866a309 100644
--- a/lib/storage/partition.go
+++ b/lib/storage/partition.go
@@ -117,8 +117,8 @@ type partition struct {
 	smallRowsDeleted    uint64
 	bigRowsDeleted      uint64
 
-	inmemoryAssistedMerges uint64
-	smallAssistedMerges    uint64
+	inmemoryAssistedMergesCount uint64
+	smallAssistedMergesCount    uint64
 
 	mergeIdx uint64
 
@@ -350,8 +350,8 @@ type partitionMetrics struct {
 	SmallPartsRefCount    uint64
 	BigPartsRefCount      uint64
 
-	InmemoryAssistedMerges uint64
-	SmallAssistedMerges    uint64
+	InmemoryAssistedMergesCount uint64
+	SmallAssistedMergesCount    uint64
 }
 
 // TotalRowsCount returns total number of rows in tm.
@@ -415,8 +415,8 @@ func (pt *partition) UpdateMetrics(m *partitionMetrics) {
 	m.SmallRowsDeleted += atomic.LoadUint64(&pt.smallRowsDeleted)
 	m.BigRowsDeleted += atomic.LoadUint64(&pt.bigRowsDeleted)
 
-	m.InmemoryAssistedMerges += atomic.LoadUint64(&pt.inmemoryAssistedMerges)
-	m.SmallAssistedMerges += atomic.LoadUint64(&pt.smallAssistedMerges)
+	m.InmemoryAssistedMergesCount += atomic.LoadUint64(&pt.inmemoryAssistedMergesCount)
+	m.SmallAssistedMergesCount += atomic.LoadUint64(&pt.smallAssistedMergesCount)
 }
 
 // AddRows adds the given rows to the partition pt.
@@ -641,7 +641,7 @@ func (pt *partition) assistedMergeForInmemoryParts() {
 		return
 	}
 
-	atomic.AddUint64(&pt.inmemoryAssistedMerges, 1)
+	atomic.AddUint64(&pt.inmemoryAssistedMergesCount, 1)
 	err := pt.mergeInmemoryParts()
 	if err == nil {
 		return
@@ -660,7 +660,7 @@ func (pt *partition) assistedMergeForSmallParts() {
 		return
 	}
 
-	atomic.AddUint64(&pt.smallAssistedMerges, 1)
+	atomic.AddUint64(&pt.smallAssistedMergesCount, 1)
 	err := pt.mergeExistingParts(false)
 	if err == nil {
 		return


Test patch:
diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go
index 2891a79b0..3a6931fdb 100644
--- a/lib/promscrape/discovery/kubernetes/pod_test.go
+++ b/lib/promscrape/discovery/kubernetes/pod_test.go
@@ -80,6 +80,17 @@ const testPodsList = `
           }
         ],
         "containers": [
+	  {
+            "name": "terminated-container",
+            "image": "terminated-image",
+            "ports": [
+              {
+                "name": "terminated-port",
+                "containerPort": 4321,
+                "protocol": "TCP"
+              }
+	    ]
+          },
           {
             "name": "etcd",
             "image": "k8s.gcr.io/etcd:3.4.3-0",
@@ -197,6 +208,15 @@ const testPodsList = `
         ],
         "startTime": "2020-03-20T13:30:29Z",
         "containerStatuses": [
+	  {
+            "name": "terminated-container",
+            "state": {
+              "terminated": {
+                "exitCode": 432
+              }
+            },
+	    "containerID": "terminated-container-id"
+          },
           {
             "name": "etcd",
             "state": {