Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4686 (2023-07-21 08:55:41)

app/vlinsert/loki: manually remove bloat dependecies for generate proto file
Reduces binary size from 22M to 14M.
Tested with promtail, those changes don't affect logs ingestion.

Thanks to @f41gh7 


Patch:
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index c331f4ab2..70347e365 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -6,16 +6,12 @@ updates:
       interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/"
-    schedule:
-      interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "bundler"
     directory: "/docs"
-    schedule:
-      interval: "daily"
+    open-pull-requests-limit: 0
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
-    schedule:
-      interval: "weekly"
     open-pull-requests-limit: 0
   - package-ecosystem: "docker"
     directory: "/"
@@ -23,6 +19,4 @@ updates:
       interval: "daily"
   - package-ecosystem: "npm"
     directory: "/app/vmui/packages/vmui"
-    schedule:
-      interval: "weekly"
     open-pull-requests-limit: 0
diff --git a/app/vlinsert/loki/push_request.pb.go b/app/vlinsert/loki/push_request.pb.go
index ad180366f..cc71eff28 100644
--- a/app/vlinsert/loki/push_request.pb.go
+++ b/app/vlinsert/loki/push_request.pb.go
@@ -7,106 +7,26 @@
 package loki
 
 import (
-	"context"
 	"fmt"
 	"io"
-	"math"
 	math_bits "math/bits"
 	"reflect"
 	"strings"
 	"time"
 
-	_ "github.com/gogo/protobuf/gogoproto"
-	proto "github.com/gogo/protobuf/proto"
-	_ "github.com/gogo/protobuf/types"
 	github_com_gogo_protobuf_types "github.com/gogo/protobuf/types"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-var _ = time.Kitchen
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
-
 type PushRequest struct {
 	Streams []Stream `protobuf:"bytes,1,rep,name=streams,proto3,customtype=Stream" json:"streams"`
 }
 
-func (m *PushRequest) Reset()      { *m = PushRequest{} }
-func (*PushRequest) ProtoMessage() {}
-func (*PushRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_35ec442956852c9e, []int{0}
-}
-func (m *PushRequest) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *PushRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_PushRequest.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *PushRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PushRequest.Merge(m, src)
-}
-func (m *PushRequest) XXX_Size() int {
-	return m.Size()
-}
-func (m *PushRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_PushRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_PushRequest proto.InternalMessageInfo
+func (m *PushRequest) Reset() { *m = PushRequest{} }
 
 type PushResponse struct {
 }
 
-func (m *PushResponse) Reset()      { *m = PushResponse{} }
-func (*PushResponse) ProtoMessage() {}
-func (*PushResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_35ec442956852c9e, []int{1}
-}
-func (m *PushResponse) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *PushResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_PushResponse.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *PushResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PushResponse.Merge(m, src)
-}
-func (m *PushResponse) XXX_Size() int {
-	return m.Size()
-}
-func (m *PushResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_PushResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_PushResponse proto.InternalMessageInfo
+func (m *PushResponse) Reset() { *m = PushResponse{} }
 
 type StreamAdapter struct {
 	Labels  string         `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
@@ -115,37 +35,7 @@ type StreamAdapter struct {
 	Hash uint64 `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
 }
 
-func (m *StreamAdapter) Reset()      { *m = StreamAdapter{} }
-func (*StreamAdapter) ProtoMessage() {}
-func (*StreamAdapter) Descriptor() ([]byte, []int) {
-	return fileDescriptor_35ec442956852c9e, []int{2}
-}
-func (m *StreamAdapter) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *StreamAdapter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_StreamAdapter.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *StreamAdapter) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StreamAdapter.Merge(m, src)
-}
-func (m *StreamAdapter) XXX_Size() int {
-	return m.Size()
-}
-func (m *StreamAdapter) XXX_DiscardUnknown() {
-	xxx_messageInfo_StreamAdapter.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_StreamAdapter proto.InternalMessageInfo
+func (m *StreamAdapter) Reset() { *m = StreamAdapter{} }
 
 func (m *StreamAdapter) GetLabels() string {
 	if m != nil {
@@ -173,37 +63,7 @@ type EntryAdapter struct {
 	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
 }
 
-func (m *EntryAdapter) Reset()      { *m = EntryAdapter{} }
-func (*EntryAdapter) ProtoMessage() {}
-func (*EntryAdapter) Descriptor() ([]byte, []int) {
-	return fileDescriptor_35ec442956852c9e, []int{3}
-}
-func (m *EntryAdapter) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *EntryAdapter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_EntryAdapter.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *EntryAdapter) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EntryAdapter.Merge(m, src)
-}
-func (m *EntryAdapter) XXX_Size() int {
-	return m.Size()
-}
-func (m *EntryAdapter) XXX_DiscardUnknown() {
-	xxx_messageInfo_EntryAdapter.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_EntryAdapter proto.InternalMessageInfo
+func (m *EntryAdapter) Reset() { *m = EntryAdapter{} }
 
 func (m *EntryAdapter) GetTimestamp() time.Time {
 	if m != nil {
@@ -219,46 +79,6 @@ func (m *EntryAdapter) GetLine() string {
 	return ""
 }
 
-func init() {
-	proto.RegisterType((*PushRequest)(nil), "logproto.PushRequest")
-	proto.RegisterType((*PushResponse)(nil), "logproto.PushResponse")
-	proto.RegisterType((*StreamAdapter)(nil), "logproto.StreamAdapter")
-	proto.RegisterType((*EntryAdapter)(nil), "logproto.EntryAdapter")
-}
-
-func init() { proto.RegisterFile("pkg/push/push.proto", fileDescriptor_35ec442956852c9e) }
-
-var fileDescriptor_35ec442956852c9e = []byte{
-	// 422 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x52, 0x41, 0x6f, 0xd3, 0x30,
-	0x18, 0xb5, 0xbb, 0xd2, 0x6d, 0xee, 0x18, 0x92, 0x61, 0xa3, 0x44, 0xc8, 0xae, 0x72, 0xea, 0x85,
-	0x44, 0x2a, 0x07, 0xce, 0x8d, 0x84, 0xb4, 0x23, 0x0a, 0x08, 0x24, 0x6e, 0x0e, 0x78, 0x4e, 0xb4,
-	0x24, 0x0e, 0xb1, 0x83, 0xc4, 0x8d, 0x9f, 0x30, 0xfe, 0x05, 0x3f, 0x65, 0xc7, 0x1e, 0x27, 0x0e,
-	0x81, 0xa6, 0x17, 0x94, 0xd3, 0x7e, 0x02, 0x8a, 0x13, 0xd3, 0xc2, 0xc5, 0x79, 0x7e, 0xfe, 0xfc,
-	0xbd, 0xf7, 0xbd, 0x18, 0x3d, 0x2c, 0xae, 0x84, 0x5f, 0x54, 0x2a, 0x36, 0x8b, 0x57, 0x94, 0x52,
-	0x4b, 0x7c, 0x94, 0x4a, 0x61, 0x90, 0xf3, 0x48, 0x48, 0x21, 0x0d, 0xf4, 0x3b, 0xd4, 0x9f, 0x3b,
-	0x54, 0x48, 0x29, 0x52, 0xee, 0x9b, 0x5d, 0x54, 0x5d, 0xfa, 0x3a, 0xc9, 0xb8, 0xd2, 0x2c, 0x2b,
-	0xfa, 0x02, 0xf7, 0x1d, 0x9a, 0xbe, 0xaa, 0x54, 0x1c, 0xf2, 0x4f, 0x15, 0x57, 0x1a, 0x5f, 0xa0,
-	0x43, 0xa5, 0x4b, 0xce, 0x32, 0x35, 0x83, 0xf3, 0x83, 0xc5, 0x74, 0xf9, 0xd8, 0xb3, 0x0a, 0xde,
-	0x6b, 0x73, 0xb0, 0xfa, 0xc8, 0x0a, 0xcd, 0xcb, 0xe0, 0xec, 0x47, 0x4d, 0x27, 0x3d, 0xd5, 0xd6,
-	0xd4, 0xde, 0x0a, 0x2d, 0x70, 0x4f, 0xd1, 0x49, 0xdf, 0x58, 0x15, 0x32, 0x57, 0xdc, 0xfd, 0x06,
-	0xd1, 0xfd, 0x7f, 0x3a, 0x60, 0x17, 0x4d, 0x52, 0x16, 0xf1, 0xb4, 0x93, 0x82, 0x8b, 0xe3, 0x00,
-	0xb5, 0x35, 0x1d, 0x98, 0x70, 0xf8, 0xe2, 0x15, 0x3a, 0xe4, 0xb9, 0x2e, 0x13, 0xae, 0x66, 0x23,
-	0xe3, 0xe7, 0x7c, 0xe7, 0xe7, 0x65, 0xae, 0xcb, 0x2f, 0xd6, 0xce, 0x83, 0x9b, 0x9a, 0x82, 0xce,
-	0xc8, 0x50, 0x1e, 0x5a, 0x80, 0x9f, 0xa0, 0x71, 0xcc, 0x54, 0x3c, 0x3b, 0x98, 0xc3, 0xc5, 0x38,
-	0xb8, 0xd7, 0xd6, 0x14, 0x3e, 0x0b, 0x0d, 0xe5, 0x7e, 0x46, 0x27, 0xfb, 0x4d, 0xf0, 0x05, 0x3a,
-	0xfe, 0x9b, 0x8f, 0x31, 0x35, 0x5d, 0x3a, 0x5e, 0x9f, 0xa0, 0x67, 0x13, 0xf4, 0xde, 0xd8, 0x8a,
-	0xe0, 0x74, 0xd0, 0x1c, 0x69, 0x75, 0xfd, 0x93, 0xc2, 0x70, 0x77, 0x19, 0x3f, 0x45, 0xe3, 0x34,
-	0xc9, 0xf9, 0x6c, 0x64, 0x26, 0x3b, 0x6a, 0x6b, 0x6a, 0xf6, 0xa1, 0x59, 0x97, 0x2b, 0x34, 0xe9,
-	0xb2, 0xe1, 0x25, 0x7e, 0x81, 0xc6, 0x1d, 0xc2, 0x67, 0xbb, 0xb1, 0xf6, 0x7e, 0x87, 0x73, 0xfe,
-	0x3f, 0x3d, 0x84, 0x09, 0x82, 0xb7, 0xeb, 0x0d, 0x01, 0xb7, 0x1b, 0x02, 0xee, 0x36, 0x04, 0x7e,
-	0x6d, 0x08, 0xfc, 0xde, 0x10, 0x78, 0xd3, 0x10, 0xb8, 0x6e, 0x08, 0xfc, 0xd5, 0x10, 0xf8, 0xbb,
-	0x21, 0xe0, 0xae, 0x21, 0xf0, 0x7a, 0x4b, 0xc0, 0x7a, 0x4b, 0xc0, 0xed, 0x96, 0x80, 0xf7, 0x73,
-	0x91, 0xe8, 0xb8, 0x8a, 0xbc, 0x0f, 0x32, 0xf3, 0x45, 0xc9, 0x2e, 0x59, 0xce, 0xfc, 0x54, 0x5e,
-	0x25, 0xbe, 0x7d, 0x5b, 0xd1, 0xc4, 0xa8, 0x3d, 0xff, 0x13, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x30,
-	0xfc, 0xe9, 0x6e, 0x02, 0x00, 0x00,
-}
-
 func (this *PushRequest) Equal(that interface{}) bool {
 	if that == nil {
 		return this == nil
@@ -428,86 +248,6 @@ func valueToGoStringPush(v interface{}, typ string) string {
 	return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
 }
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// PusherClient is the client API for Pusher service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type PusherClient interface {
-	Push(ctx context.Context, in *PushRequest, opts ...grpc.CallOption) (*PushResponse, error)
-}
-
-type pusherClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewPusherClient(cc *grpc.ClientConn) PusherClient {
-	return &pusherClient{cc}
-}
-
-func (c *pusherClient) Push(ctx context.Context, in *PushRequest, opts ...grpc.CallOption) (*PushResponse, error) {
-	out := new(PushResponse)
-	err := c.cc.Invoke(ctx, "/logproto.Pusher/Push", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// PusherServer is the server API for Pusher service.
-type PusherServer interface {
-	Push(context.Context, *PushRequest) (*PushResponse, error)
-}
-
-// UnimplementedPusherServer can be embedded to have forward compatible implementations.
-type UnimplementedPusherServer struct {
-}
-
-func (*UnimplementedPusherServer) Push(ctx context.Context, req *PushRequest) (*PushResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Push not implemented")
-}
-
-func RegisterPusherServer(s *grpc.Server, srv PusherServer) {
-	s.RegisterService(&_Pusher_serviceDesc, srv)
-}
-
-func _Pusher_Push_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(PushRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(PusherServer).Push(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/logproto.Pusher/Push",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(PusherServer).Push(ctx, req.(*PushRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Pusher_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "logproto.Pusher",
-	HandlerType: (*PusherServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Push",
-			Handler:    _Pusher_Push_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "pkg/push/push.proto",
-}
-
 func (m *PushRequest) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
diff --git a/app/vmalert/datasource/vm_graphite_api.go b/app/vmalert/datasource/vm_graphite_api.go
index 4ad5c2a6c..bb4acfeca 100644
--- a/app/vmalert/datasource/vm_graphite_api.go
+++ b/app/vmalert/datasource/vm_graphite_api.go
@@ -54,6 +54,16 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	from := "-5min"
+	if s.lookBack > 0 {
+		lookBack := timestamp.Add(-s.lookBack)
+		from = strconv.FormatInt(lookBack.Unix(), 10)
+	}
+	q.Set("from", from)
+	q.Set("format", "json")
+	q.Set("target", query)
+	q.Set("until", "now")
+
 	for k, vs := range s.extraParams {
 		if q.Has(k) { // extraParams are prior to params in URL
 			q.Del(k)
@@ -62,14 +72,6 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 			q.Add(k, v)
 		}
 	}
-	q.Set("format", "json")
-	q.Set("target", query)
-	from := "-5min"
-	if s.lookBack > 0 {
-		lookBack := timestamp.Add(-s.lookBack)
-		from = strconv.FormatInt(lookBack.Unix(), 10)
-	}
-	q.Set("from", from)
-	q.Set("until", "now")
+
 	r.URL.RawQuery = q.Encode()
 }
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index 08151f3b5..d7381a9f9 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -662,6 +662,9 @@ func registerStorageMetrics(strg *storage.Storage) {
 	metrics.NewGauge(`vm_timeseries_repopulated_total`, func() float64 {
 		return float64(m().TimeseriesRepopulated)
 	})
+	metrics.NewGauge(`vm_timeseries_precreated_total`, func() float64 {
+		return float64(m().TimeseriesPreCreated)
+	})
 	metrics.NewGauge(`vm_new_timeseries_created_total`, func() float64 {
 		return float64(m().NewTimeseriesCreated)
 	})
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5b2db0cbe..12bbb83e0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -27,7 +27,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 * SECURITY: upgrade base docker image (alpine) from 3.18.0 to 3.18.2. See [alpine 3.18.2 release notes](https://alpinelinux.org/posts/Alpine-3.15.9-3.16.6-3.17.4-3.18.2-released.html).
 * SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved).
 
-* FEATURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) for details.
+* FEATURE: reduce memory usage by up to 5x for setups with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) and long [retention](https://docs.victoriametrics.com/#retention). See [the description for this change](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7094fa38bc207c7bd7330ea8a834310a310ce5e3) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4563) for details.
+* FEATURE: reduce spikes in CPU and disk IO usage during `indexdb` rotation (aka inverted index), which is performed once per [`-retentionPeriod`](https://docs.victoriametrics.com/#retention). The new algorithm gradually pre-populates newly created `indexdb` during the last hour before the rotation. The number of pre-populated series in the newly created `indexdb` can be [monitored](https://docs.victoriametrics.com/#monitoring) via `vm_timeseries_precreated_total` metric. This should resolve [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow selecting time series matching at least one of multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}` selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels. This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) without the need to use [subqueries](https://docs.victoriametrics.com/MetricsQL.html#subqueries). See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to preserve metric names for binary operation results via `keep_metric_names` modifier. For example, `({__name__=~"foo|bar"} / 10) keep_metric_names` leaves `foo` and `bar` metric names in division results. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#keep_metric_names). This helps to address issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3710).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add ability to copy all the labels from `one` side of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches) by specifying `*` inside `group_left()` or `group_right()`. Also allow adding a prefix for copied label names via `group_left(*) prefix "..."` syntax. For example, the following query copies Kubernetes namespace labels to `kube_pod_info` series and adds `ns_` prefix for the copied label names: `kube_pod_info * on(namespace) group_left(*) prefix "ns_" kube_namespace_labels`. The labels from `on()` list aren't prefixed. This feature resolves [this](https://stackoverflow.com/questions/76661818/how-to-add-namespace-labels-to-pod-labels-in-prometheus) and [that](https://stackoverflow.com/questions/76653997/how-can-i-make-a-new-copy-of-kube-namespace-labels-metric-with-a-different-name) questions at StackOverflow.
@@ -47,6 +48,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components
 * FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow disabling of `step` param attached to [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query). This might be useful for using vmalert with datasources that to not support this param, unlike VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4573) for details.
 * FEATUTE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support option for "blackholing" alerting notifications if `-notifier.blackhole` cmd-line flag is set. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4122) for details. Thanks to @venkatbvc for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4639).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add unit test for alerting and recording rules, see more [details](https://docs.victoriametrics.com/vmalert.html#unit-testing-for-rules) here. Thanks to @Haleygo for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4596).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow overriding default GET params for rules  with `graphite` datasource type, in the same way as it happens for `prometheus` type. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4685). 
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_duration_seconds` and `vmauth_unauthorized_user_request_duration_seconds` summary metrics for measuring requests latency per user.
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): show backup progress percentage in log during backup uploading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460).
 * FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): show restoring progress percentage in log during backup downloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4460).
diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md
index b574e0268..2f59fda7e 100644
--- a/docs/MetricsQL.md
+++ b/docs/MetricsQL.md
@@ -469,7 +469,7 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
 
-`increase_pure(series_selector[d])` iis a [rollup function](#rollup-functions), which works the same as [increase](#increase) except
+`increase_pure(series_selector[d])` is a [rollup function](#rollup-functions), which works the same as [increase](#increase) except
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyConcepts.html#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 515470611..2a28b53e0 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -96,9 +96,6 @@ type indexDB struct {
 	// and is used for syncing items from different indexDBs
 	generation uint64
 
-	// The unix timestamp in seconds for the indexDB rotation.
-	rotationTimestamp uint64
-
 	name string
 	tb   *mergeset.Table
 
@@ -136,10 +133,7 @@ func getTagFiltersCacheSize() int {
 //
 // The last segment of the path should contain unique hex value which
 // will be then used as indexDB.generation
-//
-// The rotationTimestamp must be set to the current unix timestamp when mustOpenIndexDB
-// is called when creating new indexdb during indexdb rotation.
-func mustOpenIndexDB(path string, s *Storage, rotationTimestamp uint64, isReadOnly *uint32) *indexDB {
+func mustOpenIndexDB(path string, s *Storage, isReadOnly *uint32) *indexDB {
 	if s == nil {
 		logger.Panicf("BUG: Storage must be nin-nil")
 	}
@@ -157,11 +151,10 @@ func mustOpenIndexDB(path string, s *Storage, rotationTimestamp uint64, isReadOn
 	tagFiltersCacheSize := getTagFiltersCacheSize()
 
 	db := &indexDB{
-		refCount:          1,
-		generation:        gen,
-		rotationTimestamp: rotationTimestamp,
-		tb:                tb,
-		name:              name,
+		refCount:   1,
+		generation: gen,
+		tb:         tb,
+		name:       name,
 
 		tagFiltersToMetricIDsCache: workingsetcache.New(tagFiltersCacheSize),
 		s:                          s,
diff --git a/lib/storage/storage.go b/lib/storage/storage.go
index f42f3bc1d..9986191ee 100644
--- a/lib/storage/storage.go
+++ b/lib/storage/storage.go
@@ -45,6 +45,7 @@ type Storage struct {
 	tooBigTimestampRows   uint64
 
 	timeseriesRepopulated  uint64
+	timeseriesPreCreated   uint64
 	newTimeseriesCreated   uint64
 	slowRowInserts         uint64
 	slowPerDayIndexInserts uint64
@@ -53,6 +54,13 @@ type Storage struct {
 	hourlySeriesLimitRowsDropped uint64
 	dailySeriesLimitRowsDropped  uint64
 
+	// nextRotationTimestamp is a timestamp in seconds of the next indexdb rotation.
+	//
+	// It is used for gradual pre-population of the idbNext during the last hour before the indexdb rotation.
+	// in order to reduce spikes in CPU and disk IO usage just after the rotiation.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401
+	nextRotationTimestamp int64
+
 	path           string
 	cachePath      string
 	retentionMsecs int64
@@ -60,8 +68,17 @@ type Storage struct {
 	// lock file for exclusive access to the storage on the given path.
 	flockF *os.File
 
+	// idbCurr contains the currently used indexdb.
 	idbCurr atomic.Pointer[indexDB]
 
+	// idbNext is the next indexdb, which will become idbCurr at the next rotation.
+	//
+	// It is started to be gradually pre-populated with the data for active time series during the last hour
+	// before nextRotationTimestamp.
+	// This reduces spikes in CPU and disk IO usage just after the rotiation.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401
+	idbNext atomic.Pointer[indexDB]
+
 	tb *table
 
 	// Series cardinality limiters.
@@ -77,7 +94,8 @@ type Storage struct {
 	// metricNameCache is MetricID -> MetricName cache.
 	metricNameCache *workingsetcache.Cache
 
-	// dateMetricIDCache is (Date, MetricID) cache.
+	// dateMetricIDCache is (generation, Date, MetricID) cache, where generation is the indexdb generation.
+	// See generationTSID for details.
 	dateMetricIDCache *dateMetricIDCache
 
 	// Fast cache for MetricID values occurred during the current hour.
@@ -203,9 +221,6 @@ func MustOpenStorage(path string, retentionMsecs int64, maxHourlySeries, maxDail
 	s.prevHourMetricIDs.Store(hmPrev)
 	s.pendingHourEntries = &uint64set.Set{}
 
-	date := fasttime.UnixDate()
-	nextDayMetricIDs := s.mustLoadNextDayMetricIDs(date)
-	s.nextDayMetricIDs.Store(nextDayMetricIDs)
 	s.pendingNextDayMetricIDs = &uint64set.Set{}
 
 	s.prefetchedMetricIDs.Store(&uint64set.Set{})
@@ -221,9 +236,23 @@ func MustOpenStorage(path string, retentionMsecs int64, maxHourlySeries, maxDail
 	idbSnapshotsPath := filepath.Join(idbPath, snapshotsDirname)
 	fs.MustMkdirIfNotExist(idbSnapshotsPath)
 	fs.MustRemoveTemporaryDirs(idbSnapshotsPath)
-	idbCurr, idbPrev := s.mustOpenIndexDBTables(idbPath)
+	idbNext, idbCurr, idbPrev := s.mustOpenIndexDBTables(idbPath)
+
 	idbCurr.SetExtDB(idbPrev)
+	idbNext.SetExtDB(idbCurr)
+
 	s.idbCurr.Store(idbCurr)
+	s.idbNext.Store(idbNext)
+
+	// Initialize nextRotationTimestamp
+	nowSecs := time.Now().UnixNano() / 1e9
+	nextRotationTimestamp := nextRetentionDeadlineSeconds(nowSecs, retentionMsecs/1000, retentionTimezoneOffsetSecs)
+	atomic.StoreInt64(&s.nextRotationTimestamp, nextRotationTimestamp)
+
+	// Load nextDayMetricIDs cache
+	date := fasttime.UnixDate()
+	nextDayMetricIDs := s.mustLoadNextDayMetricIDs(idbCurr.generation, date)
+	s.nextDayMetricIDs.Store(nextDayMetricIDs)
 
 	// Load deleted metricIDs from idbCurr and idbPrev
 	dmisCurr, err := idbCurr.loadDeletedMetricIDs()
@@ -446,6 +475,7 @@ type Metrics struct {
 	TooBigTimestampRows   uint64
 
 	TimeseriesRepopulated  uint64
+	TimeseriesPreCreated   uint64
 	NewTimeseriesCreated   uint64
 	SlowRowInserts         uint64
 	SlowPerDayIndexInserts uint64
@@ -517,6 +547,7 @@ func (s *Storage) UpdateMetrics(m *Metrics) {
 	m.TooBigTimestampRows += atomic.LoadUint64(&s.tooBigTimestampRows)
 
 	m.TimeseriesRepopulated += atomic.LoadUint64(&s.timeseriesRepopulated)
+	m.TimeseriesPreCreated += atomic.LoadUint64(&s.timeseriesPreCreated)
 	m.NewTimeseriesCreated += atomic.LoadUint64(&s.newTimeseriesCreated)
 	m.SlowRowInserts += atomic.LoadUint64(&s.slowRowInserts)
 	m.SlowPerDayIndexInserts += atomic.LoadUint64(&s.slowPerDayIndexInserts)
@@ -587,12 +618,20 @@ func (s *Storage) UpdateMetrics(m *Metrics) {
 	m.PrefetchedMetricIDsSize += uint64(prefetchedMetricIDs.Len())
 	m.PrefetchedMetricIDsSizeBytes += uint64(prefetchedMetricIDs.SizeBytes())
 
-	m.NextRetentionSeconds = uint64(nextRetentionDuration(s.retentionMsecs).Seconds())
+	d := s.nextRetentionSeconds()
+	if d < 0 {
+		d = 0
+	}
+	m.NextRetentionSeconds = uint64(d)
 
 	s.idb().UpdateMetrics(&m.IndexDBMetrics)
 	s.tb.UpdateMetrics(&m.TableMetrics)
 }
 
+func (s *Storage) nextRetentionSeconds() int64 {
+	return atomic.LoadInt64(&s.nextRotationTimestamp) - int64(fasttime.UnixTimestamp())
+}
+
 // SetFreeDiskSpaceLimit sets the minimum free disk space size of current storage path
 //
 // The function must be called before opening or creating any storage.
@@ -649,11 +688,11 @@ func (s *Storage) startRetentionWatcher() {
 
 func (s *Storage) retentionWatcher() {
 	for {
-		d := nextRetentionDuration(s.retentionMsecs)
+		d := s.nextRetentionSeconds()
 		select {
 		case <-s.stop:
 			return
-		case <-time.After(d):
+		case <-time.After(time.Second * time.Duration(d)):
 			s.mustRotateIndexDB()
 		}
 	}
@@ -712,23 +751,29 @@ func (s *Storage) nextDayMetricIDsUpdater() {
 }
 
 func (s *Storage) mustRotateIndexDB() {
-	// Create new indexdb table.
+	// Create new indexdb table, which will be used as idbNext
 	newTableName := nextIndexDBTableName()
 	idbNewPath := filepath.Join(s.path, indexdbDirname, newTableName)
-	rotationTimestamp := fasttime.UnixTimestamp()
-	idbNew := mustOpenIndexDB(idbNewPath, s, rotationTimestamp, &s.isReadOnly)
+	idbNew := mustOpenIndexDB(idbNewPath, s, &s.isReadOnly)
+
+	// Update nextRotationTimestamp
+	atomic.AddInt64(&s.nextRotationTimestamp, s.retentionMsecs/1000)
+
+	// Set idbNext to idbNew
+	idbNext := s.idbNext.Load()
+	idbNew.SetExtDB(idbNext)
+	s.idbNext.Store(idbNew)
 
-	// Drop extDB
+	// Set idbCurr to idbNext
 	idbCurr := s.idb()
+	s.idbCurr.Store(idbNext)
+
+	// Schedule data removal for idbPrev
 	idbCurr.doExtDB(func(extDB *indexDB) {
 		extDB.scheduleToDrop()
 	})
 	idbCurr.SetExtDB(nil)
 
-	// Start using idbNew
-	idbNew.SetExtDB(idbCurr)
-	s.idbCurr.Store(idbNew)
-
 	// Persist changes on the file system.
 	fs.MustSyncPath(s.path)
 
@@ -738,7 +783,7 @@ func (s *Storage) mustRotateIndexDB() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401
 
 	// Flush metric id caches for the current and the previous hour,
-	// since they may contain entries missing in idbNew.
+	// since they may contain entries missing in idbCurr after the rotation.
 	// This should prevent from missing data in queries when
 	// the following steps are performed for short -retentionPeriod (e.g. 1 day):
 	//
@@ -748,7 +793,7 @@ func (s *Storage) mustRotateIndexDB() {
 	//    These series are already registered in prevHourMetricIDs, so VM doesn't add per-day entries to the current indexdb.
 	// 4. Stop adding new samples for these series just before 5 UTC.
 	// 5. The next indexdb rotation is performed at 4 UTC next day.
-	//    The information about the series from step 3 disappears from indexdb, since the old indexdb from step 1 is deleted,
+	//    The information about the series added at step 3 disappears from indexdb, since the old indexdb from step 1 is deleted,
 	//    while the current indexdb doesn't contain information about the series.
 	//    So queries for the last 24 hours stop returning samples added at step 3.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698
@@ -758,13 +803,12 @@ func (s *Storage) mustRotateIndexDB() {
 	s.currHourMetricIDs.Store(&hourMetricIDs{})
 	s.prevHourMetricIDs.Store(&hourMetricIDs{})
 
-	// Flush dateMetricIDCache, so idbNew can be populated with fresh data.
-	s.dateMetricIDCache.Reset()
+	// Do not flush dateMetricIDCache, since it contains entries prefixed with idb generation.
+
+	// There is no need in resetting nextDayMetricIDs, since it contains entries prefixed with idb generation.
 
 	// Do not flush metricIDCache and metricNameCache, since all the metricIDs
 	// from prev idb remain valid after the rotation.
-
-	// There is no need in resetting nextDayMetricIDs, since it should be automatically reset every day.
 }
 
 func (s *Storage) resetAndSaveTSIDCache() {
@@ -818,11 +862,14 @@ func (s *Storage) MustClose() {
 	}
 }
 
-func (s *Storage) mustLoadNextDayMetricIDs(date uint64) *byDateMetricIDEntry {
+func (s *Storage) mustLoadNextDayMetricIDs(generation, date uint64) *byDateMetricIDEntry {
 	e := &byDateMetricIDEntry{
-		date: date,
+		k: generationDateKey{
+			generation: generation,
+			date:       date,
+		},
 	}
-	name := "next_day_metric_ids"
+	name := "next_day_metric_ids_v2"
 	path := filepath.Join(s.cachePath, name)
 	if !fs.IsPathExist(path) {
 		return e
@@ -831,12 +878,17 @@ func (s *Storage) mustLoadNextDayMetricIDs(date uint64) *byDateMetricIDEntry {
 	if err != nil {
 		logger.Panicf("FATAL: cannot read %s: %s", path, err)
 	}
-	if len(src) < 16 {
-		logger.Errorf("discarding %s, since it has broken header; got %d bytes; want %d bytes", path, len(src), 16)
+	if len(src) < 24 {
+		logger.Errorf("discarding %s, since it has broken header; got %d bytes; want %d bytes", path, len(src), 24)
 		return e
 	}
 
 	// Unmarshal header
+	generationLoaded := encoding.UnmarshalUint64(src)
+	src = src[8:]
+	if generationLoaded != generation {
+		logger.Infof("discarding %s, since it contains data for stale generation; got %d; want %d", path, generationLoaded, generation)
+	}
 	dateLoaded := encoding.UnmarshalUint64(src)
 	src = src[8:]
 	if dateLoaded != date {
@@ -898,12 +950,13 @@ func (s *Storage) mustLoadHourMetricIDs(hour uint64, name string) *hourMetricIDs
 }
 
 func (s *Storage) mustSaveNextDayMetricIDs(e *byDateMetricIDEntry) {
-	name := "next_day_metric_ids"
+	name := "next_day_metric_ids_v2"
 	path := filepath.Join(s.cachePath, name)
 	dst := make([]byte, 0, e.v.Len()*8+16)
 
 	// Marshal header
-	dst = encoding.MarshalUint64(dst, e.date)
+	dst = encoding.MarshalUint64(dst, e.k.generation)
+	dst = encoding.MarshalUint64(dst, e.k.date)
 
 	// Marshal e.v
 	dst = marshalUint64Set(dst, &e.v)
@@ -1009,30 +1062,31 @@ var saveCacheLock sync.Mutex
 // SetRetentionTimezoneOffset sets the offset, which is used for calculating the time for indexdb rotation.
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574
 func SetRetentionTimezoneOffset(offset time.Duration) {
-	retentionTimezoneOffsetMsecs = offset.Milliseconds()
+	retentionTimezoneOffsetSecs = int64(offset.Seconds())
 }
 
-var retentionTimezoneOffsetMsecs int64
+var retentionTimezoneOffsetSecs int64
 
-func nextRetentionDuration(retentionMsecs int64) time.Duration {
-	nowMsecs := time.Now().UnixNano() / 1e6
-	return nextRetentionDurationAt(nowMsecs, retentionMsecs)
-}
+func nextRetentionDeadlineSeconds(atSecs, retentionSecs, offsetSecs int64) int64 {
+	// Round retentionSecs to days. This guarantees that per-day inverted index works as expected
+	const secsPerDay = 24 * 3600
+	retentionSecs = ((retentionSecs + secsPerDay - 1) / secsPerDay) * secsPerDay
 
-func nextRetentionDurationAt(atMsecs int64, retentionMsecs int64) time.Duration {
-	// Round retentionMsecs to days. This guarantees that per-day inverted index works as expected
-	retentionMsecs = ((retentionMsecs + msecPerDay - 1) / msecPerDay) * msecPerDay
+	// Schedule the deadline to +4 hours from the next retention period start
+	// because of historical reasons - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248
+	offsetSecs -= 4 * 3600
 
-	// The effect of time zone on retention period is moved out.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574
-	deadline := ((atMsecs + retentionMsecs + retentionTimezoneOffsetMsecs - 1) / retentionMsecs) * retentionMsecs
+	// Make sure that offsetSecs doesn't exceed retentionSecs
+	offsetSecs %= retentionSecs
 
-	// Schedule the deadline to +4 hours from the next retention period start.
-	// This should prevent from possible double deletion of indexdb
-	// due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 .
-	deadline += int64(4 * 3600 * 1000)
-	deadline -= retentionTimezoneOffsetMsecs
-	return time.Duration(deadline-atMsecs) * time.Millisecond
+	// align the retention deadline to multiples of retentionSecs
+	// This makes the deadline independent of atSecs.
+	deadline := ((atSecs + offsetSecs + retentionSecs - 1) / retentionSecs) * retentionSecs
+
+	// Apply the provided offsetSecs
+	deadline -= offsetSecs
+
+	return deadline
 }
 
 // SearchMetricNames returns marshaled metric names matching the given tfss on the given tr.
@@ -1553,6 +1607,7 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 	var seriesRepopulated uint64
 
 	idb := s.idb()
+	generation := idb.generation
 	is := idb.getIndexSearch(noDeadline)
 	defer idb.putIndexSearch(is)
 	var firstWarn error
@@ -1565,7 +1620,7 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 				// Skip row, since it exceeds cardinality limit
 				continue
 			}
-			if genTSID.generation != idb.generation {
+			if genTSID.generation < generation {
 				// The found TSID is from the previous indexdb. Create it in the current indexdb.
 
 				if err := mn.UnmarshalRaw(mr.MetricNameRaw); err != nil {
@@ -1580,7 +1635,7 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 				mn.sortTags()
 
 				createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-				genTSID.generation = idb.generation
+				genTSID.generation = generation
 				s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
 				seriesRepopulated++
 			}
@@ -1610,10 +1665,10 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 				continue
 			}
 
-			if genTSID.generation != idb.generation {
+			if genTSID.generation < generation {
 				// The found TSID is from the previous indexdb. Create it in the current indexdb.
 				createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-				genTSID.generation = idb.generation
+				genTSID.generation = generation
 				seriesRepopulated++
 			}
 			s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
@@ -1631,12 +1686,15 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 		// Schedule creating TSID indexes instead of creating them synchronously.
 		// This should keep stable the ingestion rate when new time series are ingested.
 		createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-		genTSID.generation = idb.generation
+		genTSID.generation = generation
 		s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
 	}
 
 	atomic.AddUint64(&s.timeseriesRepopulated, seriesRepopulated)
 
+	// There is no need in pre-filling idbNext here, since RegisterMetricNames() is rarely called.
+	// So it is OK to register metric names in blocking manner after indexdb rotation.
+
 	if firstWarn != nil {
 		logger.Warnf("cannot create some metrics: %s", firstWarn)
 	}
@@ -1644,6 +1702,7 @@ func (s *Storage) RegisterMetricNames(qt *querytracer.Tracer, mrs []MetricRow) {
 
 func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, precisionBits uint8) error {
 	idb := s.idb()
+	generation := idb.generation
 	is := idb.getIndexSearch(noDeadline)
 	defer idb.putIndexSearch(is)
 
@@ -1727,7 +1786,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 			prevTSID = r.TSID
 			prevMetricNameRaw = mr.MetricNameRaw
 
-			if genTSID.generation != idb.generation {
+			if genTSID.generation < generation {
 				// The found TSID is from the previous indexdb. Create it in the current indexdb.
 				date := uint64(r.Timestamp) / msecPerDay
 
@@ -1741,7 +1800,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 				mn.sortTags()
 
 				createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-				genTSID.generation = idb.generation
+				genTSID.generation = generation
 				s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
 				seriesRepopulated++
 				slowInsertsCount++
@@ -1775,10 +1834,10 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 				continue
 			}
 
-			if genTSID.generation != idb.generation {
+			if genTSID.generation < generation {
 				// The found TSID is from the previous indexdb. Create it in the current indexdb.
 				createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-				genTSID.generation = idb.generation
+				genTSID.generation = generation
 				seriesRepopulated++
 			}
 			s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
@@ -1799,7 +1858,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 		}
 
 		createAllIndexesForMetricName(is, mn, &genTSID.TSID, date)
-		genTSID.generation = idb.generation
+		genTSID.generation = generation
 		s.putSeriesToCache(mr.MetricNameRaw, &genTSID, date)
 		newSeriesCount++
 
@@ -1816,11 +1875,17 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 	atomic.AddUint64(&s.newTimeseriesCreated, newSeriesCount)
 	atomic.AddUint64(&s.timeseriesRepopulated, seriesRepopulated)
 
+	dstMrs = dstMrs[:j]
+	rows = rows[:j]
+
+	if err := s.prefillNextIndexDB(rows, dstMrs); err != nil {
+		if firstWarn == nil {
+			firstWarn = err
+		}
+	}
 	if firstWarn != nil {
 		storageAddRowsLogger.Warnf("warn occurred during rows addition: %s", firstWarn)
 	}
-	dstMrs = dstMrs[:j]
-	rows = rows[:j]
 
 	err := s.updatePerDateData(rows, dstMrs)
 	if err != nil {
@@ -1855,9 +1920,9 @@ func (s *Storage) putSeriesToCache(metricNameRaw []byte, genTSID *generationTSID
 	// so future rows for that TSID are ingested via fast path.
 	s.putTSIDToCache(genTSID, metricNameRaw)
 
-	// Register the (date, metricID) entry in the cache,
+	// Register the (generation, date, metricID) entry in the cache,
 	// so next time the entry is found there instead of searching for it in the indexdb.
-	s.dateMetricIDCache.Set(date, genTSID.TSID.MetricID)
+	s.dateMetricIDCache.Set(genTSID.generation, date, genTSID.TSID.MetricID)
 }
 
 func (s *Storage) registerSeriesCardinality(metricID uint64, metricNameRaw []byte) bool {
@@ -1896,6 +1961,77 @@ func getUserReadableMetricName(metricNameRaw []byte) string {
 	return mn.String()
 }
 
+func (s *Storage) prefillNextIndexDB(rows []rawRow, mrs []*MetricRow) error {
+	d := s.nextRetentionSeconds()
+	if d >= 3600 {
+		// Fast path: nothing to pre-fill because it is too early.
+		// The pre-fill is started during the last hour before the indexdb rotation.
+		return nil
+	}
+
+	// Slower path: less than hour left for the next indexdb rotation.
+	// Pre-populate idbNext with the increasing probability until the rotation.
+	// The probability increases from 0% to 100% proportioinally to d=[3600 .. 0].
+	pMin := float64(d) / 3600
+
+	idbNext := s.idbNext.Load()
+	generation := idbNext.generation
+	isNext := idbNext.getIndexSearch(noDeadline)
+	defer idbNext.putIndexSearch(isNext)
+
+	var firstError error
+	var genTSID generationTSID
+	mn := GetMetricName()
+	defer PutMetricName(mn)
+
+	timeseriesPreCreated := uint64(0)
+	for i := range rows {
+		r := &rows[i]
+		p := float64(uint32(fastHashUint64(r.TSID.MetricID))) / (1 << 32)
+		if p < pMin {
+			// Fast path: it is too early to pre-fill indexes for the given MetricID.
+			continue
+		}
+
+		// Check whether the given MetricID is already present in dateMetricIDCache.
+		date := uint64(r.Timestamp) / msecPerDay
+		metricID := r.TSID.MetricID
+		if s.dateMetricIDCache.Has(generation, date, metricID) {
+			// Indexes are already pre-filled.
+			continue
+		}
+
+		// Check whether the given (date, metricID) is already present in idbNext.
+		if isNext.hasDateMetricIDNoExtDB(date, metricID) {
+			// Indexes are already pre-filled at idbNext.
+			//
+			// Register the (generation, date, metricID) entry in the cache,
+			// so next time the entry is found there instead of searching for it in the indexdb.
+			s.dateMetricIDCache.Set(generation, date, metricID)
+			continue
+		}
+
+		// Slow path: pre-fill indexes in idbNext.
+		metricNameRaw := mrs[i].MetricNameRaw
+		if err := mn.UnmarshalRaw(metricNameRaw); err != nil {
+			if firstError == nil {
+				firstError = fmt.Errorf("cannot unmarshal MetricNameRaw %q: %w", metricNameRaw, err)
+			}
+			continue
+		}
+		mn.sortTags()
+
+		createAllIndexesForMetricName(isNext, mn, &r.TSID, date)
+		genTSID.TSID = r.TSID
+		genTSID.generation = generation
+		s.putSeriesToCache(metricNameRaw, &genTSID, date)
+		timeseriesPreCreated++
+	}
+	atomic.AddUint64(&s.timeseriesPreCreated, timeseriesPreCreated)
+
+	return firstError
+}
+
 func (s *Storage) updatePerDateData(rows []rawRow, mrs []*MetricRow) error {
 	var date uint64
 	var hour uint64
@@ -1906,6 +2042,10 @@ func (s *Storage) updatePerDateData(rows []rawRow, mrs []*MetricRow) error {
 		prevDate     uint64
 		prevMetricID uint64
 	)
+
+	idb := s.idb()
+	generation := idb.generation
+
 	hm := s.currHourMetricIDs.Load()
 	hmPrev := s.prevHourMetricIDs.Load()
 	hmPrevDate := hmPrev.hour / 24
@@ -1966,8 +2106,8 @@ func (s *Storage) updatePerDateData(rows []rawRow, mrs []*MetricRow) error {
 			}
 		}
 
-		// Slower path: check global cache for (date, metricID) entry.
-		if s.dateMetricIDCache.Has(date, metricID) {
+		// Slower path: check global cache for (generation, date, metricID) entry.
+		if s.dateMetricIDCache.Has(generation, date, metricID) {
 			continue
 		}
 		// Slow path: store the (date, metricID) entry in the indexDB.
@@ -2005,7 +2145,6 @@ func (s *Storage) updatePerDateData(rows []rawRow, mrs []*MetricRow) error {
 		return a.tsid.MetricID < b.tsid.MetricID
 	})
 
-	idb := s.idb()
 	is := idb.getIndexSearch(noDeadline)
 	defer idb.putIndexSearch(is)
 
@@ -2035,7 +2174,7 @@ func (s *Storage) updatePerDateData(rows []rawRow, mrs []*MetricRow) error {
 	}
 	PutMetricName(mn)
 	// The (date, metricID) entries must be added to cache only after they have been successfully added to indexDB.
-	s.dateMetricIDCache.Store(dateMetricIDsForCache)
+	s.dateMetricIDCache.Store(generation, dateMetricIDsForCache)
 	return firstError
 }
 
@@ -2069,12 +2208,6 @@ func newDateMetricIDCache() *dateMetricIDCache {
 	return &dmc
 }
 
-func (dmc *dateMetricIDCache) Reset() {
-	dmc.mu.Lock()
-	dmc.resetLocked()
-	dmc.mu.Unlock()
-}
-
 func (dmc *dateMetricIDCache) resetLocked() {
 	// Do not reset syncsCount and resetsCount
 	dmc.byDate.Store(newByDateMetricIDMap())
@@ -2102,9 +2235,9 @@ func (dmc *dateMetricIDCache) SizeBytes() uint64 {
 	return n
 }
 
-func (dmc *dateMetricIDCache) Has(date, metricID uint64) bool {
+func (dmc *dateMetricIDCache) Has(generation, date, metricID uint64) bool {
 	byDate := dmc.byDate.Load()
-	v := byDate.get(date)
+	v := byDate.get(generation, date)
 	if v.Has(metricID) {
 		// Fast path.
 		// The majority of calls must go here.
@@ -2113,7 +2246,7 @@ func (dmc *dateMetricIDCache) Has(date, metricID uint64) bool {
 
 	// Slow path. Check mutable map.
 	dmc.mu.Lock()
-	v = dmc.byDateMutable.get(date)
+	v = dmc.byDateMutable.get(generation, date)
 	ok := v.Has(metricID)
 	dmc.syncLockedIfNeeded()
 	dmc.mu.Unlock()
@@ -2126,7 +2259,7 @@ type dateMetricID struct {
 	metricID uint64
 }
 
-func (dmc *dateMetricIDCache) Store(dmids []dateMetricID) {
+func (dmc *dateMetricIDCache) Store(generation uint64, dmids []dateMetricID) {
 	var prevDate uint64
 	metricIDs := make([]uint64, 0, len(dmids))
 	dmc.mu.Lock()
@@ -2136,22 +2269,22 @@ func (dmc *dateMetricIDCache) Store(dmids []dateMetricID) {
 			continue
 		}
 		if len(metricIDs) > 0 {
-			v := dmc.byDateMutable.getOrCreate(prevDate)
+			v := dmc.byDateMutable.getOrCreate(generation, prevDate)
 			v.AddMulti(metricIDs)
 		}
 		metricIDs = append(metricIDs[:0], dmid.metricID)
 		prevDate = dmid.date
 	}
 	if len(metricIDs) > 0 {
-		v := dmc.byDateMutable.getOrCreate(prevDate)
+		v := dmc.byDateMutable.getOrCreate(generation, prevDate)
 		v.AddMulti(metricIDs)
 	}
 	dmc.mu.Unlock()
 }
 
-func (dmc *dateMetricIDCache) Set(date, metricID uint64) {
+func (dmc *dateMetricIDCache) Set(generation, date, metricID uint64) {
 	dmc.mu.Lock()
-	v := dmc.byDateMutable.getOrCreate(date)
+	v := dmc.byDateMutable.getOrCreate(generation, date)
 	v.Add(metricID)
 	dmc.mu.Unlock()
 }
@@ -2169,31 +2302,38 @@ func (dmc *dateMetricIDCache) syncLocked() {
 		// Nothing to sync.
 		return
 	}
+
+	// Merge data from byDate into byDateMutable and then atomically replace byDate with the merged data.
 	byDate := dmc.byDate.Load()
 	byDateMutable := dmc.byDateMutable
-	for date, e := range byDateMutable.m {
-		v := byDate.get(date)
+	for k, e := range byDateMutable.m {
+		v := byDate.get(k.generation, k.date)
 		if v == nil {
+			// Nothing to merge
 			continue
 		}
 		v = v.Clone()
 		v.Union(&e.v)
 		dme := &byDateMetricIDEntry{
-			date: date,
-			v:    *v,
+			k: k,
+			v: *v,
 		}
-		if date == byDateMutable.hotEntry.Load().date {
+		byDateMutable.m[k] = dme
+		he := byDateMutable.hotEntry.Load()
+		if he.k == k {
 			byDateMutable.hotEntry.Store(dme)
 		}
-		byDateMutable.m[date] = dme
 	}
-	for date, e := range byDate.m {
-		v := byDateMutable.get(date)
+	// Copy entries from byDate, which are missing in byDateMutable
+	for k, e := range byDate.m {
+		v := byDateMutable.get(k.generation, k.date)
 		if v != nil {
 			continue
 		}
-		byDateMutable.m[date] = e
+		byDateMutable.m[k] = e
 	}
+
+	// Atomically replace byDate with byDateMutable
 	dmc.byDate.Store(dmc.byDateMutable)
 	dmc.byDateMutable = newByDateMetricIDMap()
 
@@ -2206,25 +2346,34 @@ func (dmc *dateMetricIDCache) syncLocked() {
 
 type byDateMetricIDMap struct {
 	hotEntry atomic.Pointer[byDateMetricIDEntry]
-	m        map[uint64]*byDateMetricIDEntry
+	m        map[generationDateKey]*byDateMetricIDEntry
+}
+
+type generationDateKey struct {
+	generation uint64
+	date       uint64
 }
 
 func newByDateMetricIDMap() *byDateMetricIDMap {
 	dmm := &byDateMetricIDMap{
-		m: make(map[uint64]*byDateMetricIDEntry),
+		m: make(map[generationDateKey]*byDateMetricIDEntry),
 	}
 	dmm.hotEntry.Store(&byDateMetricIDEntry{})
 	return dmm
 }
 
-func (dmm *byDateMetricIDMap) get(date uint64) *uint64set.Set {
+func (dmm *byDateMetricIDMap) get(generation, date uint64) *uint64set.Set {
 	hotEntry := dmm.hotEntry.Load()
-	if hotEntry.date == date {
+	if hotEntry.k.generation == generation && hotEntry.k.date == date {
 		// Fast path
 		return &hotEntry.v
 	}
 	// Slow path
-	e := dmm.m[date]
+	k := generationDateKey{
+		generation: generation,
+		date:       date,
+	}
+	e := dmm.m[k]
 	if e == nil {
 		return nil
 	}
@@ -2232,36 +2381,41 @@ func (dmm *byDateMetricIDMap) get(date uint64) *uint64set.Set {
 	return &e.v
 }
 
-func (dmm *byDateMetricIDMap) getOrCreate(date uint64) *uint64set.Set {
-	v := dmm.get(date)
+func (dmm *byDateMetricIDMap) getOrCreate(generation, date uint64) *uint64set.Set {
+	v := dmm.get(generation, date)
 	if v != nil {
 		return v
 	}
+	k := generationDateKey{
+		generation: generation,
+		date:       date,
+	}
 	e := &byDateMetricIDEntry{
-		date: date,
+		k: k,
 	}
-	dmm.m[date] = e
+	dmm.m[k] = e
 	return &e.v
 }
 
 type byDateMetricIDEntry struct {
-	date uint64
-	v    uint64set.Set
+	k generationDateKey
+	v uint64set.Set
 }
 
 func (s *Storage) updateNextDayMetricIDs(date uint64) {
+	generation := s.idb().generation
 	e := s.nextDayMetricIDs.Load()
 	s.pendingNextDayMetricIDsLock.Lock()
 	pendingMetricIDs := s.pendingNextDayMetricIDs
 	s.pendingNextDayMetricIDs = &uint64set.Set{}
 	s.pendingNextDayMetricIDsLock.Unlock()
-	if pendingMetricIDs.Len() == 0 && e.date == date {
+	if pendingMetricIDs.Len() == 0 && e.k.generation == generation && e.k.date == date {
 		// Fast path: nothing to update.
 		return
 	}
 
 	// Slow path: union pendingMetricIDs with e.v
-	if e.date == date {
+	if e.k.generation == generation && e.k.date == date {
 		pendingMetricIDs.Union(&e.v)
 	} else {
 		// Do not add pendingMetricIDs from the previous day to the current day,
@@ -2269,9 +2423,13 @@ func (s *Storage) updateNextDayMetricIDs(date uint64) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3309
 		pendingMetricIDs = &uint64set.Set{}
 	}
+	k := generationDateKey{
+		generation: generation,
+		date:       date,
+	}
 	eNew := &byDateMetricIDEntry{
-		date: date,
-		v:    *pendingMetricIDs,
+		k: k,
+		v: *pendingMetricIDs,
 	}
 	s.nextDayMetricIDs.Store(eNew)
 }
@@ -2335,12 +2493,11 @@ func (s *Storage) putTSIDToCache(tsid *generationTSID, metricName []byte) {
 	s.tsidCache.Set(metricName, buf)
 }
 
-func (s *Storage) mustOpenIndexDBTables(path string) (curr, prev *indexDB) {
+func (s *Storage) mustOpenIndexDBTables(path string) (next, curr, prev *indexDB) {
 	fs.MustMkdirIfNotExist(path)
 	fs.MustRemoveTemporaryDirs(path)
 
-	// Search for the two most recent tables - the last one is active,
-	// the previous one contains backup data.
+	// Search for the three most recent tables - the prev, curr and next.
 	des := fs.MustReadDir(path)
 	var tableNames []string
 	for _, de := range des {
@@ -2358,37 +2515,42 @@ func (s *Storage) mustOpenIndexDBTables(path string) (curr, prev *indexDB) {
 	sort.Slice(tableNames, func(i, j int) bool {
 		return tableNames[i] < tableNames[j]
 	})
-	if len(tableNames) < 2 {
-		// Create missing tables
-		if len(tableNames) == 0 {
-			prevName := nextIndexDBTableName()
-			tableNames = append(tableNames, prevName)
-		}
+	switch len(tableNames) {
+	case 0:
+		prevName := nextIndexDBTableName()
 		currName := nextIndexDBTableName()
-		tableNames = append(tableNames, currName)
-	}
-
-	// Invariant: len(tableNames) >= 2
+		nextName := nextIndexDBTableName()
+		tableNames = append(tableNames, prevName, currName, nextName)
+	case 1:
+		currName := nextIndexDBTableName()
+		nextName := nextIndexDBTableName()
+		tableNames = append(tableNames, currName, nextName)
+	case 2:
+		nextName := nextIndexDBTableName()
+		tableNames = append(tableNames, nextName)
+	default:
+		// Remove all the tables except the last three tables.
+		for _, tn := range tableNames[:len(tableNames)-3] {
+			pathToRemove := filepath.Join(path, tn)
+			logger.Infof("removing obsolete indexdb dir %q...", pathToRemove)
+			fs.MustRemoveAll(pathToRemove)
+			logger.Infof("removed obsolete indexdb dir %q", pathToRemove)
+		}
+		fs.MustSyncPath(path)
 
-	// Remove all the tables except two last tables.
-	for _, tn := range tableNames[:len(tableNames)-2] {
-		pathToRemove := filepath.Join(path, tn)
-		logger.Infof("removing obsolete indexdb dir %q...", pathToRemove)
-		fs.MustRemoveAll(pathToRemove)
-		logger.Infof("removed obsolete indexdb dir %q", pathToRemove)
+		tableNames = tableNames[len(tableNames)-3:]
 	}
 
-	// Persist changes on the file system.
-	fs.MustSyncPath(path)
-
-	// Open the last two tables.
-	currPath := filepath.Join(path, tableNames[len(tableNames)-1])
+	// Open tables
+	nextPath := filepath.Join(path, tableNames[2])
+	currPath := filepath.Join(path, tableNames[1])
+	prevPath := filepath.Join(path, tableNames[0])
 
-	curr = mustOpenIndexDB(currPath, s, 0, &s.isReadOnly)
-	prevPath := filepath.Join(path, tableNames[len(tableNames)-2])
-	prev = mustOpenIndexDB(prevPath, s, 0, &s.isReadOnly)
+	next = mustOpenIndexDB(nextPath, s, &s.isReadOnly)
+	curr = mustOpenIndexDB(currPath, s, &s.isReadOnly)
+	prev = mustOpenIndexDB(prevPath, s, &s.isReadOnly)
 
-	return curr, prev
+	return next, curr, prev
 }
 
 var indexDBTableNameRegexp = regexp.MustCompile("^[0-9A-F]{16}$")


Test patch:
diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go
index 5011d300a..68c2f6e8f 100644
--- a/app/vmalert/datasource/vm_test.go
+++ b/app/vmalert/datasource/vm_test.go
@@ -611,6 +611,20 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
+		{
+			"graphite extra params allows to override from",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"from": {"-10m"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
 	}
 
 	for _, tc := range testCases {
diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go
index 58185d7ca..446feb803 100644
--- a/lib/storage/index_db_test.go
+++ b/lib/storage/index_db_test.go
@@ -495,7 +495,7 @@ func TestIndexDBOpenClose(t *testing.T) {
 	tableName := nextIndexDBTableName()
 	for i := 0; i < 5; i++ {
 		var isReadOnly uint32
-		db := mustOpenIndexDB(tableName, &s, 0, &isReadOnly)
+		db := mustOpenIndexDB(tableName, &s, &isReadOnly)
 		db.MustClose()
 	}
 	if err := os.RemoveAll(tableName); err != nil {
diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go
index f27079d60..19d3713b3 100644
--- a/lib/storage/storage_test.go
+++ b/lib/storage/storage_test.go
@@ -90,23 +90,25 @@ func TestDateMetricIDCacheConcurrent(t *testing.T) {
 
 func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 	type dmk struct {
-		date     uint64
-		metricID uint64
+		generation uint64
+		date       uint64
+		metricID   uint64
 	}
 	m := make(map[dmk]bool)
 	for i := 0; i < 1e5; i++ {
+		generation := uint64(i) % 2
 		date := uint64(i) % 3
 		metricID := uint64(i) % 1237
-		if !concurrent && c.Has(date, metricID) {
-			if !m[dmk{date, metricID}] {
-				return fmt.Errorf("c.Has(%d, %d) must return false, but returned true", date, metricID)
+		if !concurrent && c.Has(generation, date, metricID) {
+			if !m[dmk{generation, date, metricID}] {
+				return fmt.Errorf("c.Has(%d, %d, %d) must return false, but returned true", generation, date, metricID)
 			}
 			continue
 		}
-		c.Set(date, metricID)
-		m[dmk{date, metricID}] = true
-		if !concurrent && !c.Has(date, metricID) {
-			return fmt.Errorf("c.Has(%d, %d) must return true, but returned false", date, metricID)
+		c.Set(generation, date, metricID)
+		m[dmk{generation, date, metricID}] = true
+		if !concurrent && !c.Has(generation, date, metricID) {
+			return fmt.Errorf("c.Has(%d, %d, %d) must return true, but returned false", generation, date, metricID)
 		}
 		if i%11234 == 0 {
 			c.mu.Lock()
@@ -114,25 +116,29 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 			c.mu.Unlock()
 		}
 		if i%34323 == 0 {
-			c.Reset()
+			c.mu.Lock()
+			c.resetLocked()
+			c.mu.Unlock()
 			m = make(map[dmk]bool)
 		}
 	}
 
 	// Verify fast path after sync.
 	for i := 0; i < 1e5; i++ {
+		generation := uint64(i) % 2
 		date := uint64(i) % 3
 		metricID := uint64(i) % 123
-		c.Set(date, metricID)
+		c.Set(generation, date, metricID)
 	}
 	c.mu.Lock()
 	c.syncLocked()
 	c.mu.Unlock()
 	for i := 0; i < 1e5; i++ {
+		generation := uint64(i) % 2
 		date := uint64(i) % 3
 		metricID := uint64(i) % 123
-		if !concurrent && !c.Has(date, metricID) {
-			return fmt.Errorf("c.Has(%d, %d) must return true after sync", date, metricID)
+		if !concurrent && !c.Has(generation, date, metricID) {
+			return fmt.Errorf("c.Has(%d, %d, %d) must return true after sync", generation, date, metricID)
 		}
 	}
 
@@ -140,7 +146,9 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 	if n := c.EntriesCount(); !concurrent && n < 123 {
 		return fmt.Errorf("c.EntriesCount must return at least 123; returned %d", n)
 	}
-	c.Reset()
+	c.mu.Lock()
+	c.resetLocked()
+	c.mu.Unlock()
 	if n := c.EntriesCount(); !concurrent && n > 0 {
 		return fmt.Errorf("c.EntriesCount must return 0 after reset; returned %d", n)
 	}
@@ -408,45 +416,53 @@ func TestMetricRowMarshalUnmarshal(t *testing.T) {
 	}
 }
 
-func TestNextRetentionDuration(t *testing.T) {
-	validateRetention := func(retention int64) {
+func TestNextRetentionDeadlineSeconds(t *testing.T) {
+	f := func(currentTime string, retention, offset time.Duration, deadlineExpected string) {
 		t.Helper()
-		validateRetentionAt := func(now time.Time, retention int64) {
-			nowMsecs := now.UnixMilli()
-			d := nextRetentionDurationAt(nowMsecs, retention)
-			if d <= 0 {
-				nextTime := now.Add(d)
-				retentionHuman := time.Duration(retention) * time.Millisecond
-				t.Errorf("unexpected retention duration for retention=%s; got %s(%s); must be %s + %s; offset: %s", retentionHuman, nextTime, d, now, retentionHuman, time.Duration(retentionTimezoneOffsetMsecs)*time.Millisecond)
-			}
+
+		now, err := time.Parse(time.RFC3339, currentTime)
+		if err != nil {
+			t.Fatalf("cannot parse currentTime=%q: %s", currentTime, err)
 		}
 
-		// UTC offsets are in range [-12 hours, +14 hours].
-		// Verify that any legit combination of retention timezone and local time
-		// will return valid retention duration.
-		// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4207
-		for retentionOffset := -12; retentionOffset <= 14; retentionOffset++ {
-			SetRetentionTimezoneOffset(time.Duration(retentionOffset) * time.Hour)
-			validateRetentionAt(time.Now().UTC(), retention)
+		d := nextRetentionDeadlineSeconds(now.Unix(), int64(retention.Seconds()), int64(offset.Seconds()))
+		deadline := time.Unix(d, 0).UTC().Format(time.RFC3339)
+		if deadline != deadlineExpected {
+			t.Fatalf("unexpected deadline; got %s; want %s", deadline, deadlineExpected)
+		}
+	}
 
-			now := time.Date(2023, 4, 27, 23, 58, 0, 0, time.UTC)
-			validateRetentionAt(now, retention)
+	f("2023-07-22T12:44:35Z", 24*time.Hour, 0, "2023-07-23T04:00:00Z")
+	f("2023-07-22T03:44:35Z", 24*time.Hour, 0, "2023-07-22T04:00:00Z")
+	f("2023-07-22T04:44:35Z", 24*time.Hour, 0, "2023-07-23T04:00:00Z")
+	f("2023-07-22T23:44:35Z", 24*time.Hour, 0, "2023-07-23T04:00:00Z")
+	f("2023-07-23T03:59:35Z", 24*time.Hour, 0, "2023-07-23T04:00:00Z")
 
-			now = time.Date(2023, 4, 27, 0, 1, 0, 0, time.UTC)
-			validateRetentionAt(now, retention)
+	f("2023-07-22T12:44:35Z", 24*time.Hour, 2*time.Hour, "2023-07-23T02:00:00Z")
+	f("2023-07-22T01:44:35Z", 24*time.Hour, 2*time.Hour, "2023-07-22T02:00:00Z")
+	f("2023-07-22T02:44:35Z", 24*time.Hour, 2*time.Hour, "2023-07-23T02:00:00Z")
+	f("2023-07-22T23:44:35Z", 24*time.Hour, 2*time.Hour, "2023-07-23T02:00:00Z")
+	f("2023-07-23T01:59:35Z", 24*time.Hour, 2*time.Hour, "2023-07-23T02:00:00Z")
 
-			now = time.Date(2023, 4, 27, 0, 0, 0, 0, time.UTC)
-			validateRetentionAt(now, retention)
-		}
-	}
+	f("2023-07-22T12:44:35Z", 24*time.Hour, -5*time.Hour, "2023-07-23T09:00:00Z")
+	f("2023-07-22T08:44:35Z", 24*time.Hour, -5*time.Hour, "2023-07-22T09:00:00Z")
+	f("2023-07-22T09:44:35Z", 24*time.Hour, -5*time.Hour, "2023-07-23T09:00:00Z")
 
-	for retentionDays := 0.3; retentionDays < 3; retentionDays += 0.3 {
-		validateRetention(int64(retentionDays * msecPerDay))
-	}
+	f("2023-07-22T12:44:35Z", 24*time.Hour, -12*time.Hour, "2023-07-22T16:00:00Z")
+	f("2023-07-22T15:44:35Z", 24*time.Hour, -12*time.Hour, "2023-07-22T16:00:00Z")
+	f("2023-07-22T16:44:35Z", 24*time.Hour, -12*time.Hour, "2023-07-23T16:00:00Z")
 
-	for retentionMonths := float64(0.1); retentionMonths < 120; retentionMonths += 0.3 {
-		validateRetention(int64(retentionMonths * msecsPerMonth))
-	}
+	f("2023-07-22T12:44:35Z", 24*time.Hour, -18*time.Hour, "2023-07-22T22:00:00Z")
+	f("2023-07-22T21:44:35Z", 24*time.Hour, -18*time.Hour, "2023-07-22T22:00:00Z")
+	f("2023-07-22T22:44:35Z", 24*time.Hour, -18*time.Hour, "2023-07-23T22:00:00Z")
+
+	f("2023-07-22T12:44:35Z", 24*time.Hour, 18*time.Hour, "2023-07-23T10:00:00Z")
+	f("2023-07-22T09:44:35Z", 24*time.Hour, 18*time.Hour, "2023-07-22T10:00:00Z")
+	f("2023-07-22T10:44:35Z", 24*time.Hour, 18*time.Hour, "2023-07-23T10:00:00Z")
+
+	f("2023-07-22T12:44:35Z", 24*time.Hour, 37*time.Hour, "2023-07-22T15:00:00Z")
+	f("2023-07-22T14:44:35Z", 24*time.Hour, 37*time.Hour, "2023-07-22T15:00:00Z")
+	f("2023-07-22T15:44:35Z", 24*time.Hour, 37*time.Hour, "2023-07-23T15:00:00Z")
 }
 
 func TestStorageOpenClose(t *testing.T) {