Repository: VictoriaMetrics/VictoriaMetrics
Pull request #7624 (2024-11-21 17:43:53)

vmalert: fix alert states restoration
Previously, when the alert got resolved shortly before the vmalert process shuts down, this could result in false alerts.


hagen1778: One of the downsides of this new function is that Prometheus doesn't support it.

Patch:
diff --git a/app/vmalert/remoteread/init.go b/app/vmalert/remoteread/init.go
index b134dad42..b583bcfdb 100644
--- a/app/vmalert/remoteread/init.go
+++ b/app/vmalert/remoteread/init.go
@@ -14,7 +14,7 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
diff --git a/app/vmalert/rule/alerting.go b/app/vmalert/rule/alerting.go
index 1c0e4b284..11daaa329 100644
--- a/app/vmalert/rule/alerting.go
+++ b/app/vmalert/rule/alerting.go
@@ -711,7 +711,8 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 	for k, v := range ar.Labels {
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}
-	expr := fmt.Sprintf("last_over_time(%s{%s%s}[%ds])",
+	// use `default_rollup()` instead of `last_over_time()` here to accounts for possible staleness markers
+	expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
 		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
 
 	res, _, err := q.Query(ctx, expr, ts)
diff --git a/deployment/docker/vm-datasource/docker-compose-cluster.yml b/deployment/docker/vm-datasource/docker-compose-cluster.yml
index 4c216be48..9672b9a71 100644
--- a/deployment/docker/vm-datasource/docker-compose-cluster.yml
+++ b/deployment/docker/vm-datasource/docker-compose-cluster.yml
@@ -16,6 +16,6 @@ services:
       - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
       - ./../../dashboards/vm/vmauth.json:/var/lib/grafana/dashboards/vmauth.json
     environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.1/victoriametrics-datasource-v0.10.1.zip;victoriametrics-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource"
       - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
     restart: always
diff --git a/deployment/docker/vm-datasource/docker-compose.yml b/deployment/docker/vm-datasource/docker-compose.yml
index 6a77baf86..75efca834 100644
--- a/deployment/docker/vm-datasource/docker-compose.yml
+++ b/deployment/docker/vm-datasource/docker-compose.yml
@@ -15,7 +15,7 @@ services:
       - ./../../dashboards/vm/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
       - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
     environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.1/victoriametrics-datasource-v0.10.1.zip;victoriametrics-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource"
       - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
     networks:
       - vm_net
diff --git a/docs/VictoriaLogs/vmalert.md b/docs/VictoriaLogs/vmalert.md
index 0654cec22..a1931d70d 100644
--- a/docs/VictoriaLogs/vmalert.md
+++ b/docs/VictoriaLogs/vmalert.md
@@ -58,7 +58,7 @@ The following are key flags related to integration with VictoriaLogs:
 -remoteWrite.url string
    Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
 -remoteRead.url string
-   Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
+   Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
 -rule array
    Path to the files or http url with alerting and/or recording rules in YAML format.
    Supports hierarchical patterns and regexpes.
diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md
index 4b53cc626..afaabdec6 100644
--- a/docs/changelog/CHANGELOG.md
+++ b/docs/changelog/CHANGELOG.md
@@ -27,6 +27,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent): Properly return `200 OK` HTTP status code when importing data via [Pushgateway protocol](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format) using [multitenant URL format](https://docs.victoriametrics.com/cluster-victoriametrics/#url-format). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7571).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly return result for binary operation `^` aka pow at query requests for `NaN` values. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7359) for details.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix rendering of isolated data points on the graph that are not connected to other points.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): improve the correctness of alert [state restoration](https://docs.victoriametrics.com/vmalert/#alerts-state-on-restarts). Previously, it could result in false-positive alerts if alert was resolved shortly before vmalert restart.
 
 ## [v1.106.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.1)
 
diff --git a/docs/victoriametrics-datasource.md b/docs/victoriametrics-datasource.md
index 1be93a663..67a7d1f10 100644
--- a/docs/victoriametrics-datasource.md
+++ b/docs/victoriametrics-datasource.md
@@ -124,7 +124,7 @@ Please find the example of provisioning Grafana instance with VictoriaMetrics da
        grafana:
          image: grafana/grafana:11.0.0
          environment:
-         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.2/victoriametrics-datasource-v0.10.2.zip;victoriametrics-datasource
+         - GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
          - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource
          ports:
          - 3000:3000/tcp
@@ -152,14 +152,14 @@ Option 1. Using Grafana provisioning:
 
 ``` yaml
 env:
-  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.2/victoriametrics-datasource-v0.10.2.zip;victoriametrics-datasource"
+  GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource"
 ```
 
 Option 2. Using Grafana plugins section in `values.yaml`:
 
 ``` yaml
 plugins:
-  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.2/victoriametrics-datasource-v0.10.2.zip;victoriametrics-datasource
+  - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.10.3/victoriametrics-datasource-v0.10.3.zip;victoriametrics-datasource
 ```
 
 Option 3. Using init container:
diff --git a/docs/vmalert.md b/docs/vmalert.md
index 1ef85b93b..49e916353 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -85,7 +85,7 @@ Then configure `vmalert` accordingly:
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
     -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
-    -remoteRead.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource to restore alerts state from
+    -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
     -external.label=cluster=east-1 \         # External label to be applied for each rule
     -external.label=replica=a                # Multiple external labels may be set
 ```
@@ -1354,7 +1354,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
      Optional TLS server name to use for connections to -remoteRead.url. By default, the server name from -remoteRead.url is used
   -remoteRead.url string
-     Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
+     Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
   -remoteWrite.basicAuth.password string
      Optional basic auth password for -remoteWrite.url
   -remoteWrite.basicAuth.passwordFile string


Test patch:
diff --git a/app/vmalert/rule/alerting_test.go b/app/vmalert/rule/alerting_test.go
index d54aefd34..061cd2faf 100644
--- a/app/vmalert/rule/alerting_test.go
+++ b/app/vmalert/rule/alerting_test.go
@@ -791,7 +791,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with state restore
 	ts := time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
@@ -804,7 +804,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// two rules, two active alerts, one with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
 		stateMetric("bar", ts))
 	fn(
 		[]config.Rule{
@@ -824,9 +824,9 @@ func TestGroup_Restore(t *testing.T) {
 
 	// two rules, two active alerts, two with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
 		stateMetric("bar", ts))
 	fn(
 		[]config.Rule{
@@ -846,7 +846,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert but wrong state restore
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
 		stateMetric("wrong alert", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
@@ -859,7 +859,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with labels
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
@@ -872,7 +872,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with restore labels missmatch
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev", "team", "foo"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
diff --git a/apptest/client.go b/apptest/client.go
index bf63ed0b8..00926348d 100644
--- a/apptest/client.go
+++ b/apptest/client.go
@@ -1,6 +1,7 @@
 package apptest
 
 import (
+	"bytes"
 	"io"
 	"net/http"
 	"net/url"
@@ -36,13 +37,13 @@ func (c *Client) CloseConnections() {
 // the response body to the caller.
 func (c *Client) Get(t *testing.T, url string, wantStatusCode int) string {
 	t.Helper()
-	return c.do(t, http.MethodGet, url, "", "", wantStatusCode)
+	return c.do(t, http.MethodGet, url, "", nil, wantStatusCode)
 }
 
 // Post sends a HTTP POST request. Once the function receives a response, it
 // checks whether the response status code matches the expected one and returns
 // the response body to the caller.
-func (c *Client) Post(t *testing.T, url, contentType, data string, wantStatusCode int) string {
+func (c *Client) Post(t *testing.T, url, contentType string, data []byte, wantStatusCode int) string {
 	t.Helper()
 	return c.do(t, http.MethodPost, url, contentType, data, wantStatusCode)
 }
@@ -52,16 +53,16 @@ func (c *Client) Post(t *testing.T, url, contentType, data string, wantStatusCod
 // matches the expected one and returns the response body to the caller.
 func (c *Client) PostForm(t *testing.T, url string, data url.Values, wantStatusCode int) string {
 	t.Helper()
-	return c.Post(t, url, "application/x-www-form-urlencoded", data.Encode(), wantStatusCode)
+	return c.Post(t, url, "application/x-www-form-urlencoded", []byte(data.Encode()), wantStatusCode)
 }
 
 // do prepares a HTTP request, sends it to the server, receives the response
 // from the server, ensures then response code matches the expected one, reads
 // the rentire response body and returns it to the caller.
-func (c *Client) do(t *testing.T, method, url, contentType, data string, wantStatusCode int) string {
+func (c *Client) do(t *testing.T, method, url, contentType string, data []byte, wantStatusCode int) string {
 	t.Helper()
 
-	req, err := http.NewRequest(method, url, strings.NewReader(data))
+	req, err := http.NewRequest(method, url, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("could not create a HTTP request: %v", err)
 	}
diff --git a/apptest/model.go b/apptest/model.go
index f7a1902d3..2477a57e3 100644
--- a/apptest/model.go
+++ b/apptest/model.go
@@ -8,10 +8,13 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
 type PrometheusQuerier interface {
+	PrometheusAPIV1Export(t *testing.T, query, start, end string, opts QueryOpts) *PrometheusAPIV1QueryResponse
 	PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
 	PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
 	PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
@@ -19,6 +22,7 @@ type PrometheusQuerier interface {
 
 // PrometheusWriter contains methods available to Prometheus-like HTTP API for Writing new data
 type PrometheusWriter interface {
+	PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
 	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
 }
 
diff --git a/apptest/testcase.go b/apptest/testcase.go
index f98998a67..2de70db96 100644
--- a/apptest/testcase.go
+++ b/apptest/testcase.go
@@ -54,6 +54,17 @@ func (tc *TestCase) Stop() {
 	}
 }
 
+// MustStartDefaultVmsingle is a test helper function that starts an instance of
+// vmsingle with defaults suitable for most tests.
+func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
+	tc.t.Helper()
+
+	return tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmsingle",
+		"-retentionPeriod=100y",
+	})
+}
+
 // MustStartVmsingle is a test helper function that starts an instance of
 // vmsingle and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
@@ -118,7 +129,8 @@ func (c *vmcluster) ForceFlush(t *testing.T) {
 	}
 }
 
-// MustStartCluster is a typical cluster configuration.
+// MustStartDefaultCluster is a typical cluster configuration suitable for most
+// tests.
 //
 // The cluster consists of two vmstorages, one vminsert and one vmselect, no
 // data replication.
@@ -128,7 +140,7 @@ func (c *vmcluster) ForceFlush(t *testing.T) {
 // vmselect) but instead just need a typical cluster configuration to verify
 // some business logic (such as API surface, or MetricsQL). Such cluster
 // tests usually come paired with corresponding vmsingle tests.
-func (tc *TestCase) MustStartCluster() PrometheusWriteQuerier {
+func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {
 	tc.t.Helper()
 
 	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
diff --git a/apptest/tests/key_concepts_test.go b/apptest/tests/key_concepts_test.go
index 635d01ab1..8eeec4ef1 100644
--- a/apptest/tests/key_concepts_test.go
+++ b/apptest/tests/key_concepts_test.go
@@ -33,10 +33,7 @@ func TestSingleKeyConceptsQuery(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 
-	sut := tc.MustStartVmsingle("vmsingle", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage",
-		"-retentionPeriod=100y",
-	})
+	sut := tc.MustStartDefaultVmsingle()
 
 	testKeyConceptsQueryData(t, sut)
 }
@@ -47,7 +44,7 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 
-	sut := tc.MustStartCluster()
+	sut := tc.MustStartDefaultCluster()
 
 	testKeyConceptsQueryData(t, sut)
 }
diff --git a/apptest/tests/metricsql_test.go b/apptest/tests/metricsql_test.go
new file mode 100644
index 000000000..935ff5a2a
--- /dev/null
+++ b/apptest/tests/metricsql_test.go
@@ -0,0 +1,119 @@
+package tests
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func millis(s string) int64 {
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		panic(fmt.Sprintf("could not parse time %q: %v", s, err))
+	}
+	return t.UnixMilli()
+}
+
+var staleNaNsData = func() []pb.TimeSeries {
+	return []pb.TimeSeries{
+		{
+			Labels: []pb.Label{
+				{
+					Name:  "__name__",
+					Value: "metric",
+				},
+			},
+			Samples: []pb.Sample{
+				{
+					Value:     1,
+					Timestamp: millis("2024-01-01T00:01:00Z"),
+				},
+				{
+					Value:     decimal.StaleNaN,
+					Timestamp: millis("2024-01-01T00:02:00Z"),
+				},
+			},
+		},
+	}
+}()
+
+func TestSingleInstantQueryDoesNotReturnStaleNaNs(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultVmsingle()
+
+	testInstantQueryDoesNotReturnStaleNaNs(t, sut)
+}
+
+func TestClusterInstantQueryDoesNotReturnStaleNaNs(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultCluster()
+
+	testInstantQueryDoesNotReturnStaleNaNs(t, sut)
+}
+
+func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+	opts := apptest.QueryOpts{Timeout: "5s", Tenant: "0"}
+
+	sut.PrometheusAPIV1Write(t, staleNaNsData, opts)
+	sut.ForceFlush(t)
+
+	var got, want *apptest.PrometheusAPIV1QueryResponse
+	cmpOptions := []cmp.Option{
+		cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		cmpopts.EquateNaNs(),
+	}
+
+	// Verify that instant query returns the first point.
+
+	got = sut.PrometheusAPIV1Query(t, "metric", "2024-01-01T00:01:00.000Z", "5m", opts)
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
+	want.Data.Result[0].Sample = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that instant query does not return stale NaN.
+
+	got = sut.PrometheusAPIV1Query(t, "metric", "2024-01-01T00:02:00.000Z", "5m", opts)
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": []}}`)
+	// Empty response, stale NaN is not included into response
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that instant query with default rollup function returns stale NaN
+	// while it must not.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5806
+
+	got = sut.PrometheusAPIV1Query(t, "metric[2m]", "2024-01-01T00:02:00.000Z", "5m", opts)
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	s := make([]*apptest.Sample, 2)
+	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
+	want.Data.Result[0].Samples = s
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that exported data contains stale NaN.
+
+	got = sut.PrometheusAPIV1Export(t, `{__name__="metric"}`, "2024-01-01T00:01:00.000Z", "2024-01-01T00:02:00.000Z", opts)
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	s = make([]*apptest.Sample, 2)
+	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
+	want.Data.Result[0].Samples = s
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+}
diff --git a/apptest/vminsert.go b/apptest/vminsert.go
index 93fd653db..82f291345 100644
--- a/apptest/vminsert.go
+++ b/apptest/vminsert.go
@@ -7,6 +7,9 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
 )
 
 // Vminsert holds the state of a vminsert app and provides vminsert-specific
@@ -46,6 +49,20 @@ func StartVminsert(instance string, flags []string, cli *Client) (*Vminsert, err
 	}, nil
 }
 
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vminsert endpoint.
+func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.Tenant)
+	wr := pb.WriteRequest{Timeseries: records}
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	app.sendBlocking(t, len(records), func() {
+		app.cli.Post(t, url, "application/x-protobuf", data, http.StatusNoContent)
+	})
+}
+
 // PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
 // collection of records in Prometheus text exposition format for the given
 // tenant by sending a HTTP POST request to
@@ -56,9 +73,10 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.Tenant)
-	wantRowsSentCount := app.rpcRowsSentTotal(t) + len(records)
-	app.cli.Post(t, url, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
-	app.waitUntilSent(t, wantRowsSentCount)
+	data := []byte(strings.Join(records, "\n"))
+	app.sendBlocking(t, len(records), func() {
+		app.cli.Post(t, url, "text/plain", data, http.StatusNoContent)
+	})
 }
 
 // String returns the string representation of the vminsert app state.
@@ -66,21 +84,29 @@ func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
 }
 
-// waitUntilSent waits until vminsert sends buffered data to vmstorage.
+// sendBlocking sends the data to vmstorage by executing `send` function and
+// waits until the data is actually sent.
+//
+// vminsert does not send the data immediately. It first puts the data into a
+// buffer. Then a background goroutine takes the data from the buffer sends it
+// to the vmstorage. This happens every 200ms.
 //
 // Waiting is implemented a retrieving the value of `vm_rpc_rows_sent_total`
 // metric and checking whether it is equal or greater than the wanted value.
 // If it is, then the data has been sent to vmstorage.
 //
 // Unreliable if the records are inserted concurrently.
-func (app *Vminsert) waitUntilSent(t *testing.T, wantRowsSentCount int) {
+// TODO(rtm0): Put sending and waiting into a critical section to make reliable?
+func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
 	t.Helper()
 
+	send()
+
 	const (
 		retries = 20
 		period  = 100 * time.Millisecond
 	)
-
+	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
 	for range retries {
 		if app.rpcRowsSentTotal(t) >= wantRowsSentCount {
 			return
diff --git a/apptest/vmselect.go b/apptest/vmselect.go
index a93fc5fbe..c606b2c0a 100644
--- a/apptest/vmselect.go
+++ b/apptest/vmselect.go
@@ -55,9 +55,28 @@ func (app *Vmselect) ClusternativeListenAddr() string {
 	return app.clusternativeListenAddr
 }
 
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a HTTP POST request to
+// /prometheus/api/v1/export vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1export
+func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query, start, end string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export", app.httpListenAddr, opts.Tenant)
+	values := url.Values{}
+	values.Add("match[]", query)
+	values.Add("start", start)
+	values.Add("end", end)
+	values.Add("timeout", opts.Timeout)
+	values.Add("format", "promapi")
+	res := app.cli.PostForm(t, exportURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
 // PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
 // instant query by sending a HTTP POST request to /prometheus/api/v1/query
-// vmsingle endpoint.
+// vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/url-examples/#apiv1query
 func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
@@ -75,7 +94,7 @@ func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query, time, step string
 
 // PrometheusAPIV1QueryRange is a test helper function that performs
 // PromQL/MetricsQL range query by sending a HTTP POST request to
-// /prometheus/api/v1/query_range vmsingle endpoint.
+// /prometheus/api/v1/query_range vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/url-examples/#apiv1query_range
 func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
diff --git a/apptest/vmsingle.go b/apptest/vmsingle.go
index 03c5a5aa3..c1aa77b40 100644
--- a/apptest/vmsingle.go
+++ b/apptest/vmsingle.go
@@ -9,6 +9,9 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
 )
 
 // Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
@@ -20,11 +23,18 @@ type Vmsingle struct {
 	storageDataPath string
 	httpListenAddr  string
 
-	forceFlushURL                      string
+	// vmstorage URLs.
+	forceFlushURL string
+
+	// vminsert URLs.
 	prometheusAPIV1ImportPrometheusURL string
-	prometheusAPIV1QueryURL            string
-	prometheusAPIV1QueryRangeURL       string
-	prometheusAPIV1SeriesURL           string
+	prometheusAPIV1WriteURL            string
+
+	// vmselect URLs.
+	prometheusAPIV1ExportURL     string
+	prometheusAPIV1QueryURL      string
+	prometheusAPIV1QueryRangeURL string
+	prometheusAPIV1SeriesURL     string
 }
 
 // StartVmsingle starts an instance of vmsingle with the given flags. It also
@@ -56,6 +66,8 @@ func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, err
 
 		forceFlushURL:                      fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
 		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
+		prometheusAPIV1WriteURL:            fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
+		prometheusAPIV1ExportURL:           fmt.Sprintf("http://%s/prometheus/api/v1/export", stderrExtracts[1]),
 		prometheusAPIV1QueryURL:            fmt.Sprintf("http://%s/prometheus/api/v1/query", stderrExtracts[1]),
 		prometheusAPIV1QueryRangeURL:       fmt.Sprintf("http://%s/prometheus/api/v1/query_range", stderrExtracts[1]),
 		prometheusAPIV1SeriesURL:           fmt.Sprintf("http://%s/prometheus/api/v1/series", stderrExtracts[1]),
@@ -70,6 +82,17 @@ func (app *Vmsingle) ForceFlush(t *testing.T) {
 	app.cli.Get(t, app.forceFlushURL, http.StatusOK)
 }
 
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vmsingle endpoint.
+func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, _ QueryOpts) {
+	t.Helper()
+
+	wr := pb.WriteRequest{Timeseries: records}
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	app.cli.Post(t, app.prometheusAPIV1WriteURL, "application/x-protobuf", data, http.StatusNoContent)
+}
+
 // PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
 // collection of records in Prometheus text exposition format by sending a HTTP
 // POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
@@ -78,7 +101,26 @@ func (app *Vmsingle) ForceFlush(t *testing.T) {
 func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, _ QueryOpts) {
 	t.Helper()
 
-	app.cli.Post(t, app.prometheusAPIV1ImportPrometheusURL, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
+	data := []byte(strings.Join(records, "\n"))
+	app.cli.Post(t, app.prometheusAPIV1ImportPrometheusURL, "text/plain", data, http.StatusNoContent)
+}
+
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a HTTP POST request to
+// /prometheus/api/v1/export vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1export
+func (app *Vmsingle) PrometheusAPIV1Export(t *testing.T, query, start, end string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	values := url.Values{}
+	values.Add("match[]", query)
+	values.Add("start", start)
+	values.Add("end", end)
+	values.Add("timeout", opts.Timeout)
+	values.Add("format", "promapi")
+	res := app.cli.PostForm(t, app.prometheusAPIV1ExportURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
 }
 
 // PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL