Repository: ydb-platform/nbs
Pull request #3268: [Filestore] Implement CPU throttling for cleanup
Author: login: e673
Labels: name: large-tests
description: Launch large tests for PR
name: filestore
description: Add this label to run only cloud/filestore build and tests on PR
Created: 2025-03-26T21:34:16Z | Merged: 2025-04-23T20:39:53Z

Normally, Cleanup execution is scheduled in response to user operations that produce deletion markers.
Changing thresholds in the config may result in massive amount of deletion markers to be cleaned up.
This causes Cleanup to run incessantly for a long time (up to several hours) and it may result in performance degradation.

Solution: track the minimal amount of deletion markers since the last tablet reboot and throttle cleanup when the amount of deletion markers is expected to drop below the minimal amount.


Diff:
diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto
index 747262a58bf..13bad02b495 100644
--- a/cloud/filestore/config/storage.proto
+++ b/cloud/filestore/config/storage.proto
@@ -545,7 +545,7 @@ message TStorageConfig
     optional bool ServerWriteBackCacheEnabled = 420;
 
     // If enabled, GarbageCompactionThresholdAverage in Cleanup will be compared
-    // to the scaled ratio of DeletedMarkersCount to UsedBlocksCount instead of
+    // to the scaled ratio of DeletionMarkersCount to UsedBlocksCount instead of
     // UsedRangesCount.
     optional bool CalculateCleanupScoreBasedOnUsedBlocksCount = 421;
 
@@ -563,4 +563,14 @@ message TStorageConfig
     // In addition to the actively opened handles the tablet will also keep
     // in memory at most this number of the most recently used nodes
     optional uint64 SessionHandleOffloadedStatsCapacity = 425;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing configuration.
+    // (for example after changing cleanup threshold or other parameters)
+    optional uint32 CleanupCpuThrottlingThresholdPercentage = 426;
+
+    // EnqueueBlobIndexOpIfNeeded will be called with this interval
+    // if background operations should run but have been throttled.
+    // (e.g. cleanup operation due to CleanupCpuThrottlingThresholdPercentage)
+    optional uint32 EnqueueBlobIndexOpIfNeededScheduleInterval = 427; // in ms
 }
diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp
index 1a81443565c..ece9d20fea1 100644
--- a/cloud/filestore/libs/storage/core/config.cpp
+++ b/cloud/filestore/libs/storage/core/config.cpp
@@ -36,7 +36,10 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(FlushThreshold,                     ui32,   4_MB                      )\
     xxx(CleanupThreshold,                   ui32,   512                       )\
     xxx(CleanupThresholdAverage,            ui32,   64                        )\
-    xxx(CalculateCleanupScoreBasedOnUsedBlocksCount, bool,   false            )\
+    xxx(CleanupCpuThrottlingThresholdPercentage,                               \
+                                            ui32,   0                         )\
+    xxx(CalculateCleanupScoreBasedOnUsedBlocksCount,                           \
+                                            bool,   false                     )\
     xxx(NewCleanupEnabled,                  bool,   false                     )\
     xxx(CompactionThreshold,                ui32,   20                        )\
     xxx(GarbageCompactionThreshold,         ui32,   100                       )\
@@ -74,6 +77,8 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(BlobIndexOpsPriority,                                                  \
             NProto::EBlobIndexOpsPriority,                                     \
             NProto::BIOP_CLEANUP_FIRST                                        )\
+    xxx(EnqueueBlobIndexOpIfNeededScheduleInterval,                            \
+                                            TDuration, TDuration::Seconds(1)  )\
                                                                                \
     xxx(FlushThresholdForBackpressure,      ui32,      128_MB                 )\
     xxx(CleanupThresholdForBackpressure,    ui32,      32768                  )\
diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h
index b216f4f5748..68c3f10e089 100644
--- a/cloud/filestore/libs/storage/core/config.h
+++ b/cloud/filestore/libs/storage/core/config.h
@@ -75,6 +75,7 @@ class TStorageConfig
     ui32 GetFlushThreshold() const;
     ui32 GetCleanupThreshold() const;
     ui32 GetCleanupThresholdAverage() const;
+    ui32 GetCleanupCpuThrottlingThresholdPercentage() const;
     bool GetCalculateCleanupScoreBasedOnUsedBlocksCount() const;
     bool GetNewCleanupEnabled() const;
     ui32 GetCompactionThreshold() const;
@@ -225,6 +226,7 @@ class TStorageConfig
     bool GetGetNodeAttrBatchEnabled() const;
 
     NProto::EBlobIndexOpsPriority GetBlobIndexOpsPriority() const;
+    TDuration GetEnqueueBlobIndexOpIfNeededScheduleInterval() const;
 
     bool GetAllowFileStoreForceDestroy() const;
     bool GetAllowFileStoreDestroyWithOrphanSessions() const;
diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp
index 629add8525a..93c79c74898 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp
+++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp
@@ -585,6 +585,43 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx,
+    const TCleanupInfo& cleanupInfo) const
+{
+    if (!cleanupInfo.ShouldCleanup || cleanupInfo.IsPriority) {
+        return false;
+    }
+
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    const auto rangeCompactionStats = GetCompactionStats(cleanupInfo.RangeId);
+
+    const auto deletionMarkersCountAfterCleanup =
+        deletionMarkersCount >= rangeCompactionStats.DeletionsCount
+        ? deletionMarkersCount - rangeCompactionStats.DeletionsCount
+        : 0;
+
+    // Cleanup is to be throttled only when the number of deletion markers after
+    // cleanup will drop below the minimal observed amount of deletion markers.
+    // https://github.com/ydb-platform/nbs/pull/3268
+    const auto deletionMarkersCountThreshold =
+        GetMinDeletionMarkersCountSinceTabletStart();
+    if (deletionMarkersCountAfterCleanup >= deletionMarkersCountThreshold) {
+        return false;
+    }
+
+    const auto now = ctx.Now();
+    const double cleanupCpu =
+        Metrics.Cleanup.AverageSecondsPerSecond(now) * 100.0;
+    return cleanupCpu > Config->GetCleanupCpuThrottlingThresholdPercentage();
+}
+
 bool TIndexTabletActor::IsCloseToBackpressureThresholds(TString* message) const
 {
     auto bpThresholds = BuildBackpressureThresholds();
@@ -938,6 +975,7 @@ STFUNC(TIndexTabletActor::StateBoot)
         IgnoreFunc(TEvIndexTabletPrivate::TEvForcedRangeOperationProgress);
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodeRefsRequest);
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodesRequest);
+        IgnoreFunc(TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded);
 
         IgnoreFunc(TEvHiveProxy::TEvReassignTabletResponse);
 
@@ -986,6 +1024,9 @@ STFUNC(TIndexTabletActor::StateInit)
         HFunc(
             TEvIndexTabletPrivate::TEvLoadNodesRequest,
             HandleLoadNodesRequest);
+        HFunc(
+            TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded,
+            HandleEnqueueBlobIndexOpIfNeeded);
 
         FILESTORE_HANDLE_REQUEST(WaitReady, TEvIndexTablet)
 
@@ -1042,6 +1083,9 @@ STFUNC(TIndexTabletActor::StateWork)
         HFunc(
             TEvIndexTabletPrivate::TEvLoadNodesRequest,
             HandleLoadNodesRequest);
+        HFunc(
+            TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded,
+            HandleEnqueueBlobIndexOpIfNeeded);
 
         HFunc(TEvents::TEvWakeup, HandleWakeup);
         HFunc(TEvents::TEvPoisonPill, HandlePoisonPill);
@@ -1093,6 +1137,8 @@ STFUNC(TIndexTabletActor::StateZombie)
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodeRefsRequest);
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodesRequest);
 
+        IgnoreFunc(TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded);
+
         IgnoreFunc(TEvIndexTabletPrivate::TEvReadDataCompleted);
         IgnoreFunc(TEvIndexTabletPrivate::TEvWriteDataCompleted);
         IgnoreFunc(TEvIndexTabletPrivate::TEvAddDataCompleted);
@@ -1141,6 +1187,7 @@ STFUNC(TIndexTabletActor::StateBroken)
         IgnoreFunc(TEvIndexTabletPrivate::TEvForcedRangeOperationProgress);
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodeRefsRequest);
         IgnoreFunc(TEvIndexTabletPrivate::TEvLoadNodesRequest);
+        IgnoreFunc(TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded);
 
         HFunc(TEvents::TEvPoisonPill, HandlePoisonPill);
         HFunc(TEvTablet::TEvTabletDead, HandleTabletDead);
diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h
index 978b2d88f3b..a3c1a67ef8c 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_actor.h
+++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h
@@ -216,6 +216,11 @@ class TIndexTabletActor final
                 return Rate(now, RequestBytes, PrevRequestBytes);
             }
 
+            double AverageSecondsPerSecond(TInstant now) const
+            {
+                return Rate(now, TimeSumUs, PrevTimeSumUs) * 1e-6;
+            }
+
             ui64 AverageRequestSize() const
             {
                 const auto requestCount =
@@ -357,6 +362,7 @@ class TIndexTabletActor final
     bool UpdateLeakyBucketCountersScheduled = false;
     bool SyncSessionsScheduled = false;
     bool CleanupSessionsScheduled = false;
+    bool EnqueueBlobIndexOpIfNeededScheduled = false;
 
     TDeque<NActors::IEventHandlePtr> WaitReadyRequests;
 
@@ -485,6 +491,7 @@ class TIndexTabletActor final
     void EnqueueTruncateIfNeeded(const NActors::TActorContext& ctx);
     void EnqueueForcedRangeOperationIfNeeded(const NActors::TActorContext& ctx);
     void LoadNextCompactionMapChunkIfNeeded(const NActors::TActorContext& ctx);
+    void ScheduleEnqueueBlobIndexOpIfNeeded(const NActors::TActorContext& ctx);
 
     TVector<ui32> GenerateForceDeleteZeroCompactionRanges() const;
 
@@ -666,6 +673,9 @@ class TIndexTabletActor final
     ui32 ScaleCompactionThreshold(ui32 t) const;
     TCompactionInfo GetCompactionInfo() const;
     TCleanupInfo GetCleanupInfo() const;
+    bool ShouldThrottleCleanup(
+        const NActors::TActorContext& ctx,
+        const TCleanupInfo& cleanupInfo) const;
     bool IsCloseToBackpressureThresholds(TString* message) const;
 
     void HandleWakeup(
@@ -764,6 +774,10 @@ class TIndexTabletActor final
         const TEvIndexTabletPrivate::TEvLoadCompactionMapChunkResponse::TPtr& ev,
         const NActors::TActorContext& ctx);
 
+    void HandleEnqueueBlobIndexOpIfNeeded(
+        const TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded::TPtr& ev,
+        const NActors::TActorContext& ctx);
+
     void SendMetricsToExecutor(const NActors::TActorContext& ctx);
 
     bool HandleRequests(STFUNC_SIG);
diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
index e8f452b3e84..5337a7d75b7 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
+++ b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
@@ -324,6 +324,17 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
     const auto compactionInfo = GetCompactionInfo();
     const auto cleanupInfo = GetCleanupInfo();
 
+    const bool shouldThrottleCleanup = ShouldThrottleCleanup(ctx, cleanupInfo);
+    if (shouldThrottleCleanup) {
+        // Without user operations, EnqueueBlobIndexOpIfNeeded will not be
+        // called and cleanup will not resume after throttling.
+        // Therefore we reschedule calling EnqueueBlobIndexOpIfNeeded.
+        ScheduleEnqueueBlobIndexOpIfNeeded(ctx);
+    }
+
+    const bool shouldCleanup =
+        cleanupInfo.ShouldCleanup && !shouldThrottleCleanup;
+
     if (IsBlobIndexOpsQueueEmpty()) {
         auto blobIndexOpsPriority = Config->GetBlobIndexOpsPriority();
         TString message;
@@ -339,7 +350,7 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
         switch (blobIndexOpsPriority) {
             case NProto::BIOP_CLEANUP_FIRST: {
-                if (cleanupInfo.ShouldCleanup) {
+                if (shouldCleanup) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Cleanup);
                 } else if (compactionInfo.ShouldCompact) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Compaction);
@@ -350,7 +361,7 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
             case NProto::BIOP_COMPACTION_FIRST: {
                 if (compactionInfo.ShouldCompact) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Compaction);
-                } else if (cleanupInfo.ShouldCleanup) {
+                } else if (shouldCleanup) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Cleanup);
                 }
                 break;
@@ -360,7 +371,7 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
                 if (compactionInfo.ShouldCompact) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Compaction);
                 }
-                if (cleanupInfo.ShouldCleanup) {
+                if (shouldCleanup) {
                     AddBackgroundBlobIndexOp(EBlobIndexOp::Cleanup);
                 }
                 break;
@@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededScheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+        EnqueueBlobIndexOpIfNeededScheduled = true;
+
+        LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+            "%s EnqueueBlobIndexOpIfNeeded scheduled",
+            LogTag.c_str());
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TIndexTabletActor::HandleEnqueueBlobIndexOpIfNeeded(
+    const TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded::TPtr& ev,
+    const TActorContext& ctx)
+{
+    Y_UNUSED(ev);
+
+    LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+        "%s EnqueueBlobIndexOpIfNeeded received",
+        LogTag.c_str());
+
+    EnqueueBlobIndexOpIfNeededScheduled = false;
+    EnqueueBlobIndexOpIfNeeded(ctx);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 void TIndexTabletActor::HandleCompaction(
     const TEvIndexTabletPrivate::TEvCompactionRequest::TPtr& ev,
     const TActorContext& ctx)
diff --git a/cloud/filestore/libs/storage/tablet/tablet_private.h b/cloud/filestore/libs/storage/tablet/tablet_private.h
index 9c8e410de35..5c2ae7c77cd 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_private.h
+++ b/cloud/filestore/libs/storage/tablet/tablet_private.h
@@ -942,6 +942,8 @@ struct TEvIndexTabletPrivate
         EvLoadNodeRefs,
         EvLoadNodes,
 
+        EvEnqueueBlobIndexOpIfNeeded,
+
         EvEnd
     };
 
@@ -989,6 +991,9 @@ struct TEvIndexTabletPrivate
 
     using TEvLoadNodesRequest =
         TRequestEvent<TLoadNodesRequest, EvLoadNodes>;
+
+    using TEvEnqueueBlobIndexOpIfNeeded =
+        TRequestEvent<TEmpty, EvEnqueueBlobIndexOpIfNeeded>;
 };
 
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.cpp b/cloud/filestore/libs/storage/tablet/tablet_state.cpp
index 8ed011c0802..4949636ba1d 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_state.cpp
+++ b/cloud/filestore/libs/storage/tablet/tablet_state.cpp
@@ -121,6 +121,16 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in a massive amount of
+    // deletion markers to be cleaned up.
+    // This causes Cleanup to run incessantly for a long time (up to several
+    // hours) and it may result in performance degradation.
+    // We track the minimal amount of deletion markers since the last tablet
+    // reboot and throttle cleanup when the amount of deletion markers is
+    // expected to drop below the minimal amount.
+    // https://github.com/ydb-platform/nbs/pull/3268
+    MinDeletionMarkersCountSinceTabletStart =
+        fileSystemStats.GetDeletionMarkersCount();
 
     if (FileSystemStats.GetLastNodeId() < RootNodeId) {
         FileSystemStats.SetLastNodeId(RootNodeId);
diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h
index dae198fd95f..34cf6651e4c 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_state.h
+++ b/cloud/filestore/libs/storage/tablet/tablet_state.h
@@ -178,6 +178,7 @@ class TIndexTabletState
     NProto::TFileSystemStats FileSystemStats;
     NCloud::NProto::TTabletStorageInfo TabletStorageInfo;
     TNodeToSessionCounters NodeToSessionCounters;
+    ui64 MinDeletionMarkersCountSinceTabletStart = 0;
 
     /*const*/ ui32 TruncateBlocksThreshold = 0;
     /*const*/ ui32 SessionHistoryEntryCount = 0;
@@ -294,6 +295,19 @@ class TIndexTabletState
         return FileSystemStats;
     }
 
+    ui64 GetMinDeletionMarkersCountSinceTabletStart() const
+    {
+        return MinDeletionMarkersCountSinceTabletStart;
+    }
+
+    void UpdateMinDeletionMarkersCountSinceTabletStart()
+    {
+        MinDeletionMarkersCountSinceTabletStart = Min(
+            MinDeletionMarkersCountSinceTabletStart,
+            FileSystemStats.GetDeletionMarkersCount()
+        );
+    }
+
     const TNodeToSessionCounters& GetNodeToSessionCounters() const
     {
         return NodeToSessionCounters;
diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
index fb7b78f855b..47e90f0a606 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
+++ b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
@@ -943,6 +943,7 @@ ui32 TIndexTabletState::CleanupBlockDeletions(
     }
 
     DecrementDeletionMarkersCount(db, deletionMarkerCount);
+    UpdateMinDeletionMarkersCountSinceTabletStart();
 
     auto largeDeletionMarkers =
         Impl->LargeBlocks.ExtractProcessedDeletionMarkers();
diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
index 068977ddcfa..3e2a64766f4 100644
--- a/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
+++ b/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
@@ -649,6 +649,126 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for larger blocks the file size
+    // will be enormously higher
+    TABLET_TEST_4K_ONLY(ShouldThrottleCleanupAfterTabletRestart)
+    {
+        const auto block = tabletConfig.BlockSize;
+        const auto fileSize = 16_MB;
+        const auto blockCount = fileSize / block;
+
+        // Disable automatic cleanup
+        NProto::TStorageConfig storageConfig;
+        storageConfig.SetNewCleanupEnabled(true);
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetWriteBlobThreshold(block);
+
+        TTestEnv env({}, std::move(storageConfig));
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+        ui64 tabletId = env.BootIndexTablet(nodeIdx);
+
+        TIndexTabletClient tablet(
+            env.GetRuntime(),
+            nodeIdx,
+            tabletId,
+            tabletConfig);
+        tablet.InitSession("client", "session");
+
+        auto nodeId = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        auto handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+
+        auto args = TSetNodeAttrArgs(nodeId).SetSize(block);
+        tablet.SetNodeAttr(args);
+
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_VALUES_EQUAL(2 * blockCount - 1,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Enable automatic cleanup and configure throttling to consume 1% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThresholdPercentage(1);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        // Cleanup is expected to run a bit and stop because of throttling
+        ui64 deletionMarkersCount = 0;
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            deletionMarkersCount = stats.GetDeletionMarkersCount();
+            UNIT_ASSERT_LT(0, deletionMarkersCount);
+            UNIT_ASSERT_GT(2 * blockCount - 1, deletionMarkersCount);
+        }
+
+        // Spawn deletion markers
+        handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+        tablet.SetNodeAttr(args);
+
+        // Cleanup is expected to trottle only when the amount of
+        // deletion markers is below the threshold (2 * blockCount - 1)
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            deletionMarkersCount = stats.GetDeletionMarkersCount();
+            UNIT_ASSERT_LT(0, deletionMarkersCount);
+            UNIT_ASSERT_GT(2 * blockCount - 1, deletionMarkersCount);
+        }
+
+        // Advance time - cleanup should be rescheduled and process more blocks
+        tablet.AdvanceTime(TDuration::Minutes(1));
+        env.GetRuntime().DispatchEvents({}, TDuration::Minutes(1));
+
+        // Cleanup is expected to process a bit more blocks
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_GT(deletionMarkersCount,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Disable cleanup and spawn deletion markers again
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(false);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+        tablet.SetNodeAttr(args);
+
+        // Enable cleanup and configure throttling to consume 100% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThresholdPercentage(100);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+
+        // Cleanup is expected to process all the blocks
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_VALUES_EQUAL(0, stats.GetDeletionMarkersCount());
+        }
+    }
 }
 
 }   // namespace NCloud::NFileStore::NStorage


Reviews:
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T17:09:25Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T17:30:46Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T17:35:58Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T17:39:13Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:23:45Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:24:01Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:24:08Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:26:23Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:26:52Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T20:37:21Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T21:06:13Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T08:24:11Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T08:24:24Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T08:24:49Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T08:25:39Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T08:25:51Z
user: login: debnatkh
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T11:16:01Z
user: login: debnatkh
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T11:17:07Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T11:27:45Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:33:09Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:38:06Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:38:33Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:38:41Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:38:57Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T20:39:41Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T21:07:41Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T16:41:15Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T16:41:43Z
user: login: debnatkh0
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T00:03:48Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T07:30:34Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T07:30:41Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T07:31:34Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:39:06Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:41:53Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:42:16Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:43:13Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:43:48Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:44:17Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:35:23Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:36:18Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:36:28Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:36:37Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:36:45Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:36:52Z
user: login: e673
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T16:37:03Z
user: login: SvartMetal
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-23T19:04:23Z
user: login: yegorskii
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-23T19:30:33Z

Review comments:
diff_hunk: @@ -529,10 +529,14 @@ message TStorageConfig
     optional bool ServerWriteBackCacheEnabled = 420;
 
     // If enabled, GarbageCompactionThresholdAverage in Cleanup will be compared
-    // to the scaled ratio of DeletedMarkersCount to UsedBlocksCount instead of
+    // to the scaled ratio of DeletionMarkersCount to UsedBlocksCount instead of
     // UsedRangesCount.
     optional bool CalculateCleanupScoreBasedOnUsedBlocksCount = 421;
 
     // Use SSL for dynamic node registration.
     optional bool NodeRegistrationUseSsl = 422;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing settings.
+    optional uint32 CleanupCpuThrottlingThreshold = 423;
path: cloud/filestore/config/storage.proto
user: login: SvartMetal
body: CleanupCpuThrottlingThresholdPercentage
created_at: 2025-04-04T17:09:25Z
author_association: COLLABORATOR
diff_hunk: @@ -323,6 +323,8 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 {
     const auto compactionInfo = GetCompactionInfo();
     const auto cleanupInfo = GetCleanupInfo();
+    const bool shouldCleanup =
+        ShouldCleanupAfterApplyingThrottlingRules(ctx, cleanupInfo);
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: SvartMetal
body: может заинлайним? а то название немного обманывает, исходя из семантики функции
created_at: 2025-04-04T17:30:45Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,7 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    DeletionMarkersThrottlingThreshold = fileSystemStats.GetDeletionMarkersCount();
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: SvartMetal
body: комментарий
created_at: 2025-04-04T17:35:58Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,111 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
+    // will be enormously higher
+    TABLET_TEST_4K_ONLY(ShouldThrottleCleanupAfterTabletRestart)
+    {
+        const auto block = tabletConfig.BlockSize;
+        const auto fileSize = 16_MB;
+        const auto blockCount = fileSize / block;
+
+        // Disable automatic cleanup
+        NProto::TStorageConfig storageConfig;
+        storageConfig.SetNewCleanupEnabled(true);
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetWriteBlobThreshold(block);
+
+        TTestEnv env({}, std::move(storageConfig));
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+        ui64 tabletId = env.BootIndexTablet(nodeIdx);
+
+        TIndexTabletClient tablet(
+            env.GetRuntime(),
+            nodeIdx,
+            tabletId,
+            tabletConfig);
+        tablet.InitSession("client", "session");
+
+        auto nodeId = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        auto handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+
+        TSetNodeAttrArgs args(nodeId);
+        args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
+        args.SetSize(block);
+        tablet.SetNodeAttr(args);
+
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_VALUES_EQUAL(2 * blockCount - 1,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Enable automatic cleanup and configure throttling to consume 1% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThreshold(1);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        // Cleanup is expected to run a bit and stop because of throttling
+        ui64 deletionMarkersCount = 0;
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            deletionMarkersCount = stats.GetDeletionMarkersCount();
+            UNIT_ASSERT_LT(0, deletionMarkersCount);
+            UNIT_ASSERT_GT(2 * blockCount - 1, deletionMarkersCount);
+        }
+
+        // Advance time and trigger cleanup again by calling a user operation
+        tablet.AdvanceTime(TDuration::Seconds(1));
+        tablet.SetNodeAttr(args);
+
+        // Cleanup is expected to process a bit more blocks
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_GT(deletionMarkersCount,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Disable cleanup and spawn deletion markers again
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(false);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+        tablet.SetNodeAttr(args);
+
+        // Enable cleanup and configure throttling to consume 100% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThreshold(100);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: SvartMetal
body: хочется ещё тестировать случай когда добавилось сильно много маркеров - клинап должен срабатывать несмотря на строгий тротлинг
created_at: 2025-04-04T17:39:12Z
author_association: COLLABORATOR
diff_hunk: @@ -323,6 +323,8 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 {
     const auto compactionInfo = GetCompactionInfo();
     const auto cleanupInfo = GetCleanupInfo();
+    const bool shouldCleanup =
+        ShouldCleanupAfterApplyingThrottlingRules(ctx, cleanupInfo);
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: e673
body: Поправил
created_at: 2025-04-07T20:23:45Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,7 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    DeletionMarkersThrottlingThreshold = fileSystemStats.GetDeletionMarkersCount();
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: e673
body: Добавил
created_at: 2025-04-07T20:24:01Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,111 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
+    // will be enormously higher
+    TABLET_TEST_4K_ONLY(ShouldThrottleCleanupAfterTabletRestart)
+    {
+        const auto block = tabletConfig.BlockSize;
+        const auto fileSize = 16_MB;
+        const auto blockCount = fileSize / block;
+
+        // Disable automatic cleanup
+        NProto::TStorageConfig storageConfig;
+        storageConfig.SetNewCleanupEnabled(true);
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetWriteBlobThreshold(block);
+
+        TTestEnv env({}, std::move(storageConfig));
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+        ui64 tabletId = env.BootIndexTablet(nodeIdx);
+
+        TIndexTabletClient tablet(
+            env.GetRuntime(),
+            nodeIdx,
+            tabletId,
+            tabletConfig);
+        tablet.InitSession("client", "session");
+
+        auto nodeId = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        auto handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+
+        TSetNodeAttrArgs args(nodeId);
+        args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
+        args.SetSize(block);
+        tablet.SetNodeAttr(args);
+
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_VALUES_EQUAL(2 * blockCount - 1,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Enable automatic cleanup and configure throttling to consume 1% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThreshold(1);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        // Cleanup is expected to run a bit and stop because of throttling
+        ui64 deletionMarkersCount = 0;
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            deletionMarkersCount = stats.GetDeletionMarkersCount();
+            UNIT_ASSERT_LT(0, deletionMarkersCount);
+            UNIT_ASSERT_GT(2 * blockCount - 1, deletionMarkersCount);
+        }
+
+        // Advance time and trigger cleanup again by calling a user operation
+        tablet.AdvanceTime(TDuration::Seconds(1));
+        tablet.SetNodeAttr(args);
+
+        // Cleanup is expected to process a bit more blocks
+        {
+            auto response = tablet.GetStorageStats();
+            const auto& stats = response->Record.GetStats();
+            UNIT_ASSERT_VALUES_EQUAL(1, stats.GetUsedBlocksCount());
+            UNIT_ASSERT_GT(deletionMarkersCount,
+                stats.GetDeletionMarkersCount());
+        }
+
+        // Disable cleanup and spawn deletion markers again
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(false);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+        tablet.InitSession("client", "session");
+
+        handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+        tablet.SetNodeAttr(args);
+
+        // Enable cleanup and configure throttling to consume 100% of CPU
+        storageConfig.SetCleanupThresholdAverage(64);
+        storageConfig.SetCleanupCpuThrottlingThreshold(100);
+        storageConfig.SetCalculateCleanupScoreBasedOnUsedBlocksCount(true);
+        tablet.ChangeStorageConfig(storageConfig);
+        tablet.RebootTablet();
+
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: e673
body: Добавил
created_at: 2025-04-07T20:24:07Z
author_association: COLLABORATOR
diff_hunk: @@ -529,10 +529,14 @@ message TStorageConfig
     optional bool ServerWriteBackCacheEnabled = 420;
 
     // If enabled, GarbageCompactionThresholdAverage in Cleanup will be compared
-    // to the scaled ratio of DeletedMarkersCount to UsedBlocksCount instead of
+    // to the scaled ratio of DeletionMarkersCount to UsedBlocksCount instead of
     // UsedRangesCount.
     optional bool CalculateCleanupScoreBasedOnUsedBlocksCount = 421;
 
     // Use SSL for dynamic node registration.
     optional bool NodeRegistrationUseSsl = 422;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing configuration.
path: cloud/filestore/config/storage.proto
user: login: SvartMetal
body: (for example after changing cleanup threshold or other configuration parameters)
created_at: 2025-04-07T20:26:22Z
author_association: COLLABORATOR
diff_hunk: @@ -36,6 +36,7 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(FlushThreshold,                     ui32,   4_MB                      )\
     xxx(CleanupThreshold,                   ui32,   512                       )\
     xxx(CleanupThresholdAverage,            ui32,   64                        )\
+    xxx(CleanupCpuThrottlingThresholdPercentage, ui32,   0                    )\
path: cloud/filestore/libs/storage/core/config.cpp
user: login: SvartMetal
body: выравниванием
created_at: 2025-04-07T20:26:52Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
+    if (!Config->GetCleanupCpuThrottlingThresholdPercentage()) {
+        return false;
+    }
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: \n
created_at: 2025-04-07T20:27:21Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
+    if (!Config->GetCleanupCpuThrottlingThresholdPercentage()) {
+        return false;
+    }
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: \n
created_at: 2025-04-07T20:27:26Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
+    if (!Config->GetCleanupCpuThrottlingThresholdPercentage()) {
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: лучше явно сравнивать с чем-нибудь, это ведь не bool
created_at: 2025-04-07T20:27:54Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: точно комментарий в правильном месте, кажется больше подходит к 592 строке
created_at: 2025-04-07T20:28:44Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
+    if (!Config->GetCleanupCpuThrottlingThresholdPercentage()) {
+        return false;
+    }
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
+    const auto now = ctx.Now();
+    const double cleanupCpu = Metrics.Cleanup.CPU(now) * 100.0;
+    return cleanupCpu > Config->GetCleanupCpuThrottlingThresholdPercentage();
+}
+
+void TIndexTabletActor::UpdateDeletionMarkersThrottlingThreshold(
+    const TCleanupInfo& cleanupInfo)
+{
+    if (!cleanupInfo.ShouldCleanup) {
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: кажется логичнее иметь этот if выше
created_at: 2025-04-07T20:37:13Z
author_association: COLLABORATOR
diff_hunk: @@ -323,6 +323,9 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 {
     const auto compactionInfo = GetCompactionInfo();
     const auto cleanupInfo = GetCleanupInfo();
+    const bool shouldCleanup = cleanupInfo.ShouldCleanup &&
+        (cleanupInfo.IsPriority || !ShouldThrottleCleanup(ctx));
+    UpdateDeletionMarkersThrottlingThreshold(cleanupInfo);
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: SvartMetal
body: кажется логичнее это вызвать в конце cleanup
created_at: 2025-04-07T21:06:13Z
author_association: COLLABORATOR
diff_hunk: @@ -529,10 +529,14 @@ message TStorageConfig
     optional bool ServerWriteBackCacheEnabled = 420;
 
     // If enabled, GarbageCompactionThresholdAverage in Cleanup will be compared
-    // to the scaled ratio of DeletedMarkersCount to UsedBlocksCount instead of
+    // to the scaled ratio of DeletionMarkersCount to UsedBlocksCount instead of
     // UsedRangesCount.
     optional bool CalculateCleanupScoreBasedOnUsedBlocksCount = 421;
 
     // Use SSL for dynamic node registration.
     optional bool NodeRegistrationUseSsl = 422;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing configuration.
path: cloud/filestore/config/storage.proto
user: login: e673
body: Добавил, только убрал слово configuration (это и так понятно из предыдущей строки), а то некрасиво на следующую строчку переносится.
created_at: 2025-04-08T08:24:11Z
author_association: COLLABORATOR
diff_hunk: @@ -36,6 +36,7 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(FlushThreshold,                     ui32,   4_MB                      )\
     xxx(CleanupThreshold,                   ui32,   512                       )\
     xxx(CleanupThresholdAverage,            ui32,   64                        )\
+    xxx(CleanupCpuThrottlingThresholdPercentage, ui32,   0                    )\
path: cloud/filestore/libs/storage/core/config.cpp
user: login: e673
body: Выравнил все параметры в блоке.
created_at: 2025-04-08T08:24:24Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: e673
body: Переставил
created_at: 2025-04-08T08:24:49Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,36 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    // Throttle cleanup when the amount of deletion markers is below threshold
+    if (!Config->GetCleanupCpuThrottlingThresholdPercentage()) {
+        return false;
+    }
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
+    const auto now = ctx.Now();
+    const double cleanupCpu = Metrics.Cleanup.CPU(now) * 100.0;
+    return cleanupCpu > Config->GetCleanupCpuThrottlingThresholdPercentage();
+}
+
+void TIndexTabletActor::UpdateDeletionMarkersThrottlingThreshold(
+    const TCleanupInfo& cleanupInfo)
+{
+    if (!cleanupInfo.ShouldCleanup) {
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: e673
body: Перенёс логику функции в `TIndexTabletActor::ExecuteTx_Cleanup`.
created_at: 2025-04-08T08:25:39Z
author_association: COLLABORATOR
diff_hunk: @@ -323,6 +323,9 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 {
     const auto compactionInfo = GetCompactionInfo();
     const auto cleanupInfo = GetCleanupInfo();
+    const bool shouldCleanup = cleanupInfo.ShouldCleanup &&
+        (cleanupInfo.IsPriority || !ShouldThrottleCleanup(ctx));
+    UpdateDeletionMarkersThrottlingThreshold(cleanupInfo);
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: e673
body: Перенёс логику функции в TIndexTabletActor::ExecuteTx_Cleanup.
created_at: 2025-04-08T08:25:51Z
author_association: COLLABORATOR
diff_hunk: @@ -32,27 +32,28 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(WriteBatchTimeout,             TDuration, TDuration::MilliSeconds(0)  )\
     xxx(WriteBlobThreshold,            ui32,      128_KB                      )\
                                                                                \
-    xxx(MaxBlobSize,                        ui32,   4_MB                      )\
path: cloud/filestore/libs/storage/core/config.cpp
user: login: debnatkh
body: А зачем? Может не портить тут блейм?
created_at: 2025-04-08T10:29:57Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,12 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in massive amount of
+    // deletion markers to be cleaned up. It may result in service degradation.
+    // Cleanup is expected to throttle when the amount of deletion markers is
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: debnatkh
body: to be throttled
created_at: 2025-04-08T10:32:11Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +295,16 @@ class TIndexTabletState
         return FileSystemStats;
     }
 
+    ui64 GetDeletionMarkersThrottlingThreshold() const
path: cloud/filestore/libs/storage/tablet/tablet_state.h
user: login: debnatkh
body: А зачем тут сеттеры/геттеры?
created_at: 2025-04-08T10:41:23Z
author_association: COLLABORATOR
diff_hunk: @@ -213,6 +213,11 @@ class TIndexTabletActor final
                 return Rate(now, RequestBytes, PrevRequestBytes);
             }
 
+            double CPU(TInstant now) const
path: cloud/filestore/libs/storage/tablet/tablet_actor.h
user: login: debnatkh
body: Немного обманчивое значение. Для неоднопоточных операций типа read/unlink/etc CPU может превосходить единицу
created_at: 2025-04-08T11:10:40Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,26 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    // Throttle cleanup only when the amount of deletion markers is below threshold
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
+
+    const auto now = ctx.Now();
+    const double cleanupCpu = Metrics.Cleanup.CPU(now) * 100.0;
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: debnatkh
body: cleanupCpu будет зануляться каждый раз, когда зовётся `Metrics.Cleanup.UpdatePrev`
created_at: 2025-04-08T11:11:54Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,128 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
+    // will be enormously higher
+    TABLET_TEST_4K_ONLY(ShouldThrottleCleanupAfterTabletRestart)
+    {
+        const auto block = tabletConfig.BlockSize;
+        const auto fileSize = 16_MB;
+        const auto blockCount = fileSize / block;
+
+        // Disable automatic cleanup
+        NProto::TStorageConfig storageConfig;
+        storageConfig.SetNewCleanupEnabled(true);
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetWriteBlobThreshold(block);
+
+        TTestEnv env({}, std::move(storageConfig));
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+        ui64 tabletId = env.BootIndexTablet(nodeIdx);
+
+        TIndexTabletClient tablet(
+            env.GetRuntime(),
+            nodeIdx,
+            tabletId,
+            tabletConfig);
+        tablet.InitSession("client", "session");
+
+        auto nodeId = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        auto handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+
+        TSetNodeAttrArgs args(nodeId);
+        args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: debnatkh
body: Можно делать так:
TSetNodeAttrArgs(nodeId).SetSize(block);
created_at: 2025-04-08T11:15:05Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,128 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: debnatkh
body: larger
created_at: 2025-04-08T11:15:52Z
author_association: COLLABORATOR
diff_hunk: @@ -32,27 +32,28 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(WriteBatchTimeout,             TDuration, TDuration::MilliSeconds(0)  )\
     xxx(WriteBlobThreshold,            ui32,      128_KB                      )\
                                                                                \
-    xxx(MaxBlobSize,                        ui32,   4_MB                      )\
path: cloud/filestore/libs/storage/core/config.cpp
user: login: debnatkh
body: Вижу
https://github.com/ydb-platform/nbs/pull/3268#discussion_r2032668361
Но можно это
сделать дополнительную отюивку, чтобы не резало глаза, можно на три строки разнести этот аргумент
created_at: 2025-04-08T11:17:07Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,26 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    // Throttle cleanup only when the amount of deletion markers is below threshold
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
+
+    const auto now = ctx.Now();
+    const double cleanupCpu = Metrics.Cleanup.CPU(now) * 100.0;
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: e673
body: Да, и это ожидаемое поведение.
Фактически, получается ограничение на суммарное потребление CPU в 15-секундных интервалах (частота вызова UpdatePrev).
Для данной задачи большего и не надо.
created_at: 2025-04-08T11:27:45Z
author_association: COLLABORATOR
diff_hunk: @@ -32,27 +32,28 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(WriteBatchTimeout,             TDuration, TDuration::MilliSeconds(0)  )\
     xxx(WriteBlobThreshold,            ui32,      128_KB                      )\
                                                                                \
-    xxx(MaxBlobSize,                        ui32,   4_MB                      )\
path: cloud/filestore/libs/storage/core/config.cpp
user: login: e673
body: Вернул, сделал только для одного параметра.
created_at: 2025-04-08T20:33:09Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +295,16 @@ class TIndexTabletState
         return FileSystemStats;
     }
 
+    ui64 GetDeletionMarkersThrottlingThreshold() const
path: cloud/filestore/libs/storage/tablet/tablet_state.h
user: login: e673
body: Приватное поле, инкапсуляция, всё такое.
created_at: 2025-04-08T20:38:06Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,128 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: e673
body: Fixed
created_at: 2025-04-08T20:38:33Z
author_association: COLLABORATOR
diff_hunk: @@ -649,6 +649,128 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         const auto resp = AssertWriteDataResponse(S_OK);
         UNIT_ASSERT(resp->Record.GetHeaders().GetThrottler().GetDelay() > 0);
     }
+
+    // The test is 4K only because for largers blocks the file size
+    // will be enormously higher
+    TABLET_TEST_4K_ONLY(ShouldThrottleCleanupAfterTabletRestart)
+    {
+        const auto block = tabletConfig.BlockSize;
+        const auto fileSize = 16_MB;
+        const auto blockCount = fileSize / block;
+
+        // Disable automatic cleanup
+        NProto::TStorageConfig storageConfig;
+        storageConfig.SetNewCleanupEnabled(true);
+        storageConfig.SetCleanupThresholdAverage(999'999);
+        storageConfig.SetWriteBlobThreshold(block);
+
+        TTestEnv env({}, std::move(storageConfig));
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+        ui64 tabletId = env.BootIndexTablet(nodeIdx);
+
+        TIndexTabletClient tablet(
+            env.GetRuntime(),
+            nodeIdx,
+            tabletId,
+            tabletConfig);
+        tablet.InitSession("client", "session");
+
+        auto nodeId = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        auto handle = CreateHandle(tablet, nodeId);
+        tablet.WriteData(handle, 0, fileSize, 'a');
+        tablet.DestroyHandle(handle);
+
+        TSetNodeAttrArgs args(nodeId);
+        args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
path: cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
user: login: e673
body: Fixed
created_at: 2025-04-08T20:38:41Z
author_association: COLLABORATOR
diff_hunk: @@ -580,6 +580,26 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx) const
+{
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    // Throttle cleanup only when the amount of deletion markers is below threshold
+    if (deletionMarkersCount >= GetDeletionMarkersThrottlingThreshold()) {
+        return false;
+    }
+
+    const auto now = ctx.Now();
+    const double cleanupCpu = Metrics.Cleanup.CPU(now) * 100.0;
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: e673
body: Договорились, что так и оставим.
created_at: 2025-04-08T20:38:57Z
author_association: COLLABORATOR
diff_hunk: @@ -213,6 +213,11 @@ class TIndexTabletActor final
                 return Rate(now, RequestBytes, PrevRequestBytes);
             }
 
+            double CPU(TInstant now) const
path: cloud/filestore/libs/storage/tablet/tablet_actor.h
user: login: e673
body: Договорились об имени `AverageSecondsPerSecond`
created_at: 2025-04-08T20:39:41Z
author_association: COLLABORATOR
diff_hunk: @@ -151,6 +151,16 @@ void TIndexTabletActor::CompleteTx_Cleanup(
         1,
         args.ProcessedDeletionMarkerCount * GetBlockSize(),
         ctx.Now() - args.RequestInfo->StartedTs);
+
+    const auto cleanupInfo = GetCleanupInfo();
+    if (!cleanupInfo.ShouldCleanup) {
path: cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp
user: login: SvartMetal
body: тут бы ещё комментарий про физический смысл этого условия

что мы снижаем threshold только когда серия cleanup'ов закончилась
created_at: 2025-04-08T21:07:40Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +295,16 @@ class TIndexTabletState
         return FileSystemStats;
     }
 
+    ui64 GetDeletionMarkersThrottlingThreshold() const
path: cloud/filestore/libs/storage/tablet/tablet_state.h
user: login: e673
body: Неактуально, изменена логика
created_at: 2025-04-10T16:41:15Z
author_association: COLLABORATOR
diff_hunk: @@ -151,6 +151,16 @@ void TIndexTabletActor::CompleteTx_Cleanup(
         1,
         args.ProcessedDeletionMarkerCount * GetBlockSize(),
         ctx.Now() - args.RequestInfo->StartedTs);
+
+    const auto cleanupInfo = GetCleanupInfo();
+    if (!cleanupInfo.ShouldCleanup) {
path: cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp
user: login: e673
body: Неактульно вследствие доработки алгоритма
created_at: 2025-04-10T16:41:43Z
author_association: COLLABORATOR
diff_hunk: @@ -32,6 +32,7 @@ constexpr TDuration UpdateCountersInterval = TDuration::Seconds(15);
 constexpr TDuration UpdateLeakyBucketCountersInterval = TDuration::Seconds(1);
 constexpr TDuration UpdateStatsInterval = TDuration::Seconds(1);
 constexpr TDuration DumpTracksInterval = TDuration::Seconds(15);
+constexpr TDuration EnqueueBlobIndexOpIfNeededInterval = TDuration::Seconds(1);
path: cloud/storage/core/libs/diagnostics/public.h
user: login: debnatkh0
body: Надо сделать конфигурируемым параметром в storage-конфиге.
created_at: 2025-04-21T23:59:14Z
author_association: NONE
diff_hunk: @@ -121,6 +121,15 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in massive amount of
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: debnatkh0
body: in a massive
created_at: 2025-04-22T00:00:26Z
author_association: NONE
diff_hunk: @@ -178,6 +178,7 @@ class TIndexTabletState
     NProto::TFileSystemStats FileSystemStats;
     NCloud::NProto::TTabletStorageInfo TabletStorageInfo;
     TNodeToSessionCounters NodeToSessionCounters;
+    ui64 MinimalDeletionMarkersCount = 0;
path: cloud/filestore/libs/storage/tablet/tablet_state.h
user: login: debnatkh0
body: MinimalDeletionMarkersCountSinceTabletStart может? Или слишком многословно?
created_at: 2025-04-22T00:01:51Z
author_association: NONE
diff_hunk: @@ -943,6 +943,7 @@ ui32 TIndexTabletState::CleanupBlockDeletions(
     }
 
     DecrementDeletionMarkersCount(db, deletionMarkerCount);
+    UpdateMinimalDeletionMarkersCount();
path: cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
user: login: debnatkh0
body: https://github.com/ydb-platform/nbs/blob/dd9083265544c2aa0acb24034231fe13beb03d2c/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp#L840

А вот тут это тоже не надо ли звать?
created_at: 2025-04-22T00:03:34Z
author_association: NONE
diff_hunk: @@ -178,6 +178,7 @@ class TIndexTabletState
     NProto::TFileSystemStats FileSystemStats;
     NCloud::NProto::TTabletStorageInfo TabletStorageInfo;
     TNodeToSessionCounters NodeToSessionCounters;
+    ui64 MinimalDeletionMarkersCount = 0;
path: cloud/filestore/libs/storage/tablet/tablet_state.h
user: login: e673
body: Да вроде норм - переименовал.
created_at: 2025-04-23T07:30:34Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,15 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in massive amount of
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: e673
body: Done
created_at: 2025-04-23T07:30:41Z
author_association: COLLABORATOR
diff_hunk: @@ -943,6 +943,7 @@ ui32 TIndexTabletState::CleanupBlockDeletions(
     }
 
     DecrementDeletionMarkersCount(db, deletionMarkerCount);
+    UpdateMinimalDeletionMarkersCount();
path: cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
user: login: e673
body: Минимальное значение может только уменьшаться, поэтому после инкремента звать эту функцию не нужно.
created_at: 2025-04-23T07:31:33Z
author_association: COLLABORATOR
diff_hunk: @@ -585,6 +585,43 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx,
+    const TCleanupInfo& cleanupInfo) const
+{
+    if (!cleanupInfo.ShouldCleanup || cleanupInfo.IsPriority) {
+        return false;
+    }
+
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    const auto rangeCompactionStats = GetCompactionStats(cleanupInfo.RangeId);
+
+    const auto deletionMarkersCountAfterCleanup =
+        rangeCompactionStats.DeletionsCount <= deletionMarkersCount
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: SvartMetal
body: deletionMarkersCount >= rangeCompactionStats.DeletionsCount

кажется чуть проще читается
created_at: 2025-04-23T15:39:06Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: SvartMetal
body: лишний отступ
created_at: 2025-04-23T15:41:52Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
+
+        LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+            "%s TEvEnqueueBlobIndexOpIfNeeded scheduled",
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: SvartMetal
body: EnqueueBlobIndexOpIfNeeded scheduled
created_at: 2025-04-23T15:42:16Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
+
+        LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+            "%s TEvEnqueueBlobIndexOpIfNeeded scheduled",
+            LogTag.c_str());
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TIndexTabletActor::HandleEnqueueBlobIndexOpIfNeeded(
+    const TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded::TPtr& ev,
+    const TActorContext& ctx)
+{
+    Y_UNUSED(ev);
+
+    LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+        "%s TEvEnqueueBlobIndexOpIfNeeded received",
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: SvartMetal
body: EnqueueBlobIndexOpIfNeeded event received
created_at: 2025-04-23T15:43:13Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,16 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in a massive amount of
+    // deletion markers to be cleaned up.
+    // This causes Cleanup to run incessantly for a long time (up to several
+    // hours) and it may result in performance degradation.
+    // We track the minimal amount of deletion markers since the last tablet
+    // reboot and throttle cleanup when the amount of deletion markers is
+    // expected to drop below the minimal amount.
+    // https://github.com/ydb-platform/nbs/pull/3268
+    MinimalDeletionMarkersCountSinceTabletStart =
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: SvartMetal
body: MinDeletionMarkersCountSinceTabletStart
created_at: 2025-04-23T15:43:48Z
author_association: COLLABORATOR
diff_hunk: @@ -563,4 +563,14 @@ message TStorageConfig
     // In addition to the actively opened handles the tablet will also keep
     // in memory at most this number of the most recently used nodes
     optional uint64 SessionHandleOffloadedStatsCapacity = 425;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing configuration.
+    // (for example after changing cleanup threshold or other parameters)
+    optional uint32 CleanupCpuThrottlingThresholdPercentage = 426;
+
+    // EnqueueBlobIndexOpIfNeeded will be called with this interval
+    // if background operations should run but have been throttled.
+    // (e.g. cleanup operation due to CleanupCpuThrottlingThresholdPercentage)
+    optional uint32 EnqueueBlobIndexOpIfNeededRescheduleInterval = 427; // in ms
path: cloud/filestore/config/storage.proto
user: login: SvartMetal
body: EnqueueBlobIndexOpIfNeededScheduleInterval
created_at: 2025-04-23T15:44:17Z
author_association: COLLABORATOR
diff_hunk: @@ -585,6 +585,43 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const
             || isPriority};
 }
 
+bool TIndexTabletActor::ShouldThrottleCleanup(
+    const NActors::TActorContext& ctx,
+    const TCleanupInfo& cleanupInfo) const
+{
+    if (!cleanupInfo.ShouldCleanup || cleanupInfo.IsPriority) {
+        return false;
+    }
+
+    if (Config->GetCleanupCpuThrottlingThresholdPercentage() == 0) {
+        return false;
+    }
+
+    const auto deletionMarkersCount =
+        GetFileSystemStats().GetDeletionMarkersCount();
+
+    const auto rangeCompactionStats = GetCompactionStats(cleanupInfo.RangeId);
+
+    const auto deletionMarkersCountAfterCleanup =
+        rangeCompactionStats.DeletionsCount <= deletionMarkersCount
path: cloud/filestore/libs/storage/tablet/tablet_actor.cpp
user: login: e673
body: Fixed
created_at: 2025-04-23T16:35:23Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: e673
body: Fixed
created_at: 2025-04-23T16:36:18Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
+
+        LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+            "%s TEvEnqueueBlobIndexOpIfNeeded scheduled",
+            LogTag.c_str());
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TIndexTabletActor::HandleEnqueueBlobIndexOpIfNeeded(
+    const TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded::TPtr& ev,
+    const TActorContext& ctx)
+{
+    Y_UNUSED(ev);
+
+    LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+        "%s TEvEnqueueBlobIndexOpIfNeeded received",
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: e673
body: Fixed
created_at: 2025-04-23T16:36:28Z
author_association: COLLABORATOR
diff_hunk: @@ -563,4 +563,14 @@ message TStorageConfig
     // In addition to the actively opened handles the tablet will also keep
     // in memory at most this number of the most recently used nodes
     optional uint64 SessionHandleOffloadedStatsCapacity = 425;
+
+    // Limit CPU consumption by Cleanup when a massive amount
+    // of deletion markers is to be cleaned up after changing configuration.
+    // (for example after changing cleanup threshold or other parameters)
+    optional uint32 CleanupCpuThrottlingThresholdPercentage = 426;
+
+    // EnqueueBlobIndexOpIfNeeded will be called with this interval
+    // if background operations should run but have been throttled.
+    // (e.g. cleanup operation due to CleanupCpuThrottlingThresholdPercentage)
+    optional uint32 EnqueueBlobIndexOpIfNeededRescheduleInterval = 427; // in ms
path: cloud/filestore/config/storage.proto
user: login: e673
body: Fixed
created_at: 2025-04-23T16:36:37Z
author_association: COLLABORATOR
diff_hunk: @@ -121,6 +121,16 @@ void TIndexTabletState::LoadState(
     FileSystem.CopyFrom(fileSystem);
     FileSystemStats.CopyFrom(fileSystemStats);
     TabletStorageInfo.CopyFrom(tabletStorageInfo);
+    // Changing thresholds in the config may result in a massive amount of
+    // deletion markers to be cleaned up.
+    // This causes Cleanup to run incessantly for a long time (up to several
+    // hours) and it may result in performance degradation.
+    // We track the minimal amount of deletion markers since the last tablet
+    // reboot and throttle cleanup when the amount of deletion markers is
+    // expected to drop below the minimal amount.
+    // https://github.com/ydb-platform/nbs/pull/3268
+    MinimalDeletionMarkersCountSinceTabletStart =
path: cloud/filestore/libs/storage/tablet/tablet_state.cpp
user: login: e673
body: Fixed
created_at: 2025-04-23T16:36:45Z
author_association: COLLABORATOR
diff_hunk: @@ -426,6 +437,39 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TIndexTabletActor::ScheduleEnqueueBlobIndexOpIfNeeded(
+    const NActors::TActorContext& ctx)
+{
+    if (!EnqueueBlobIndexOpIfNeededScheduled) {
+        ctx.Schedule(
+            Config->GetEnqueueBlobIndexOpIfNeededRescheduleInterval(),
+            new TEvIndexTabletPrivate::TEvEnqueueBlobIndexOpIfNeeded());
+            EnqueueBlobIndexOpIfNeededScheduled = true;
+
+        LOG_TRACE(ctx, TFileStoreComponents::TABLET,
+            "%s TEvEnqueueBlobIndexOpIfNeeded scheduled",
path: cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
user: login: e673
body: Fixed
created_at: 2025-04-23T16:36:52Z
author_association: COLLABORATOR
diff_hunk: @@ -32,6 +32,7 @@ constexpr TDuration UpdateCountersInterval = TDuration::Seconds(15);
 constexpr TDuration UpdateLeakyBucketCountersInterval = TDuration::Seconds(1);
 constexpr TDuration UpdateStatsInterval = TDuration::Seconds(1);
 constexpr TDuration DumpTracksInterval = TDuration::Seconds(15);
+constexpr TDuration EnqueueBlobIndexOpIfNeededInterval = TDuration::Seconds(1);
path: cloud/storage/core/libs/diagnostics/public.h
user: login: e673
body: Сделано
created_at: 2025-04-23T16:37:03Z
author_association: COLLABORATOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-26T22:49:49Z
body: <!-- status pr=3268, run=10332, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 96b67fd7bd7e7aa9d3aa3f688dfad991cdcd74a3.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6543](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14093922695/1/nebius-x86-64/summary/ya-test.html) | [6543](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14093922695/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-31T10:53:19Z
body: <!-- status pr=3268, run=10442, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 753f4aadf035e0e6205b8744e3e583c664978d70.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6545](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14168648555/1/nebius-x86-64/summary/ya-test.html) | [6544](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14168648555/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14168648555/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-31T17:00:08Z
body: <!-- status pr=3268, run=10465, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 9a26017446d7bc5534af3096baaed4ac6f94ef97.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6568](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14175537297/1/nebius-x86-64/summary/ya-test.html) | [6561](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14175537297/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14175537297/1/nebius-x86-64/summary/ya-test.html#FAIL) | [4](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14175537297/1/nebius-x86-64/summary/ya-test.html#SKIP) | [2](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14175537297/1/nebius-x86-64/summary/ya-test.html#MUTE) |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-31T19:09:09Z
body: <!-- status pr=3268, run=10467, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit fd8666e0a487b71fb0f1c15245803eba762d4376.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14178110742/1/nebius-x86-64/summary/ya-test.html) | [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14178110742/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-07T21:46:56Z
body: <!-- status pr=3268, run=10686, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 7f8b2ba4ff59e51286beb4239f5cab254fabc137.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6224](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14318740484/1/nebius-x86-64/summary/ya-test.html) | [6223](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14318740484/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14318740484/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-08T09:45:11Z
body: <!-- status pr=3268, run=10699, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 20bb393037dbe0b9453ac6ddb09c448ae7b2c5dd.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14328696922/1/nebius-x86-64/summary/ya-test.html) | [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14328696922/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-08T22:08:03Z
body: <!-- status pr=3268, run=10729, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit a382c2a2c328fb14db1996f97dfde1abb08422c0.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6572](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14343024695/1/nebius-x86-64/summary/ya-test.html) | [6560](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14343024695/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | [8](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14343024695/1/nebius-x86-64/summary/ya-test.html#SKIP) | [4](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14343024695/1/nebius-x86-64/summary/ya-test.html#MUTE) |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-09T23:06:10Z
body: <!-- status pr=3268, run=10752, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit fd42101f95abfb3277e3334340fcc31106782008.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6571](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14367516868/1/nebius-x86-64/summary/ya-test.html) | [6569](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14367516868/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [2](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14367516868/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-10T18:31:50Z
body: <!-- status pr=3268, run=10771, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit dd9083265544c2aa0acb24034231fe13beb03d2c.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14385747724/1/nebius-x86-64/summary/ya-test.html) | [6570](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14385747724/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-23T08:56:40Z
body: <!-- status pr=3268, run=11019, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 402066caf26908e399ad6b49317d1cbfa1a152a2.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6617](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14612442146/1/nebius-x86-64/summary/ya-test.html) | [6616](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14612442146/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14612442146/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-23T17:58:35Z
body: <!-- status pr=3268, run=11039, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit f86d433b1951753afc3ac70f81d0a496e22983cd.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6620](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14623336568/1/nebius-x86-64/summary/ya-test.html) | [6619](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14623336568/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14623336568/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-23T20:33:45Z
body: <!-- status pr=3268, run=11045, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit f86d433b1951753afc3ac70f81d0a496e22983cd.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [2160](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14626067719/1/nebius-x86-64/summary/ya-test.html) | [2160](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14626067719/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR