Repository: ydb-platform/nbs
Pull request #3180: [Filestore] Fix throttling for GenerateBlobIds request
Author: login: proller
Created: 2025-03-12T22:27:47Z | Merged: 2025-03-24T14:31:08Z

#1709


Diff:
diff --git a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp
index 86dc656edfa..842718813f9 100644
--- a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp
+++ b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp
@@ -271,16 +271,21 @@ void TReadDataActor::HandleDescribeDataResponse(
     const TActorContext& ctx)
 {
     const auto* msg = ev->Get();
+    const auto& error = msg->GetError();
 
     TABLET_VERIFY(InFlightRequest);
 
-    InFlightRequest->Complete(ctx.Now(), msg->GetError());
+    InFlightRequest->Complete(ctx.Now(), error);
     FinalizeProfileLogRequestInfo(
         InFlightRequest->ProfileLogRequest,
         msg->Record);
 
     if (FAILED(msg->GetStatus())) {
-        ReadData(ctx, FormatError(msg->GetError()));
+        if (error.GetCode() != E_FS_THROTTLED) {
+            ReadData(ctx, FormatError(error));
+        } else {
+            HandleError(ctx, error);
+        }
         return;
     }
 
diff --git a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp
index 900095b9707..493fab559ef 100644
--- a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp
+++ b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp
@@ -157,16 +157,21 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
         const TActorContext& ctx)
     {
         const auto* msg = ev->Get();
+        const auto& error = msg->GetError();
 
         TABLET_VERIFY(InFlightRequest);
 
-        InFlightRequest->Complete(ctx.Now(), msg->GetError());
+        InFlightRequest->Complete(ctx.Now(), error);
         FinalizeProfileLogRequestInfo(
             InFlightRequest->ProfileLogRequest,
             msg->Record);
 
-        if (HasError(msg->GetError())) {
-            WriteData(ctx, msg->GetError());
+        if (HasError(error)) {
+            if (error.GetCode() != E_FS_THROTTLED) {
+                WriteData(ctx, error);
+            } else {
+                HandleError(ctx, error);
+            }
             return;
         }
 
diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp
index 20546a8302f..5ea14fee064 100644
--- a/cloud/filestore/libs/storage/service/service_ut.cpp
+++ b/cloud/filestore/libs/storage/service/service_ut.cpp
@@ -2675,7 +2675,6 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
             UNIT_ASSERT_VALUES_EQUAL(
                 response.GetStorageConfig().GetThreeStageWriteEnabled(),
                 true);
-            TDispatchOptions options;
             env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1));
         }
 
@@ -2776,6 +2775,105 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldThrottleMulipleStageReadsAndWrites)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        {
+            const auto blockCount = 1'000;
+            auto request = service.CreateCreateFileStoreRequest(fs, blockCount);
+            {
+                auto performanceProfile =
+                    request->Record.MutablePerformanceProfile();
+                performanceProfile->SetThrottlingEnabled(true);
+                performanceProfile->SetMaxWriteBandwidth(maxBandwidth);
+                performanceProfile->SetMaxReadBandwidth(maxBandwidth);
+                performanceProfile->SetMaxPostponedWeight(1);
+                performanceProfile->SetMaxPostponedTime(1);
+                performanceProfile->SetMaxPostponedCount(1);
+                performanceProfile->SetDefaultPostponedRequestWeight(1);
+            }
+
+            service.SendAndRecvCreateFileStore(std::move(request));
+        }
+        auto headers = service.InitSession(fs, "client");
+        ui64 nodeId =
+            service
+                .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file"))
+                ->Record.GetNode()
+                .GetId();
+
+        ui64 handle =
+            service
+                .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR)
+                ->Record.GetHandle();
+
+        auto& runtime = env.GetRuntime();
+
+        size_t throttled = 0;
+        auto writeReadData = [&](ui64 offset, ui64 size)
+        {
+            auto data = GenerateValidateData(size);
+            {
+                auto response = service.SendAndRecvWriteData(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data);
+                if (response->GetError().GetCode() == E_FS_THROTTLED) {
+                    ++throttled;
+                }
+            }
+            {
+                auto response = service.SendAndRecvReadData(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data.size());
+                if (response->GetError().GetCode() == E_FS_THROTTLED) {
+                    ++throttled;
+                }
+            }
+        };
+        constexpr auto size =
+            128_MB + 1;   // Config.DefaultThresholds.MaxPostponedWeight
+        for (auto i = 0; i < 5; ++i) {
+            writeReadData(size * i, size);
+        }
+
+        UNIT_ASSERT_VALUES_EQUAL(10, throttled);
+
+        UNIT_ASSERT_VALUES_EQUAL(
+            5,
+            runtime.GetCounter(TEvService::EvWriteDataRequest));
+        UNIT_ASSERT_VALUES_EQUAL(
+            10,
+            runtime.GetCounter(TEvIndexTablet::EvGenerateBlobIdsRequest));
+        UNIT_ASSERT_VALUES_EQUAL(
+            5,
+            runtime.GetCounter(TEvService::EvReadDataRequest));
+        UNIT_ASSERT_VALUES_EQUAL(
+            10,
+            runtime.GetCounter(TEvIndexTablet::EvDescribeDataRequest));
+    }
+
     Y_UNIT_TEST(ShouldSendBSGroupFlagsToTabletViaAddDataRequests)
     {
         TTestEnv env;


Reviews:
user: login: debnatkh
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T16:11:31Z
user: login: debnatkh
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T16:18:55Z
user: login: debnatkh
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-03-21T17:32:05Z
user: login: debnatkh
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-21T23:32:47Z
user: login: debnatkh
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-23T08:29:01Z
user: login: yegorskii
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-24T14:02:10Z

Review comments:
diff_hunk: @@ -271,16 +271,19 @@ void TReadDataActor::HandleDescribeDataResponse(
     const TActorContext& ctx)
 {
     const auto* msg = ev->Get();
+    const auto& error = msg->GetError();
 
     TABLET_VERIFY(InFlightRequest);
 
-    InFlightRequest->Complete(ctx.Now(), msg->GetError());
+    InFlightRequest->Complete(ctx.Now(), error);
     FinalizeProfileLogRequestInfo(
         InFlightRequest->ProfileLogRequest,
         msg->Record);
 
     if (FAILED(msg->GetStatus())) {
-        ReadData(ctx, FormatError(msg->GetError()));
+        if (error.GetCode() != E_FS_THROTTLED) {
+            ReadData(ctx, FormatError(error));
+        }
path: cloud/filestore/libs/storage/service/service_actor_readdata.cpp
user: login: debnatkh
body: В такой реализации у нас будут утекать акторы. Если describe завершился тротлингом, нам надо отвечать дальше REJECT, а не тихо умирать в этом месте
created_at: 2025-03-18T16:11:28Z
author_association: COLLABORATOR
diff_hunk: @@ -271,16 +271,19 @@ void TReadDataActor::HandleDescribeDataResponse(
     const TActorContext& ctx)
 {
     const auto* msg = ev->Get();
+    const auto& error = msg->GetError();
 
     TABLET_VERIFY(InFlightRequest);
 
-    InFlightRequest->Complete(ctx.Now(), msg->GetError());
+    InFlightRequest->Complete(ctx.Now(), error);
     FinalizeProfileLogRequestInfo(
         InFlightRequest->ProfileLogRequest,
         msg->Record);
 
     if (FAILED(msg->GetStatus())) {
-        ReadData(ctx, FormatError(msg->GetError()));
+        if (error.GetCode() != E_FS_THROTTLED) {
+            ReadData(ctx, FormatError(error));
+        }
path: cloud/filestore/libs/storage/service/service_actor_readdata.cpp
user: login: debnatkh
body: Короче, тут нужно 
```
if (error.GetCode() != E_FS_THROTTLED) {
    ReadData(ctx, FormatError(error));
} else {
    HandleError....
}
```
created_at: 2025-03-18T16:18:55Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
+
+        {
+            auto resizeRequest =
+                service.CreateResizeFileStoreRequest("test", blockCount);
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: ```suggestion
                service.CreateResizeFileStoreRequest("test", blockCount);
                auto& performanceProfile = resizeRequest->Record.MutablePerformanceProfile();
```
created_at: 2025-03-21T17:25:06Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
+
+        {
+            auto resizeRequest =
+                service.CreateResizeFileStoreRequest("test", blockCount);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetThrottlingEnabled(true);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxWriteBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxReadBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedWeight(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedTime(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedCount(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetDefaultPostponedRequestWeight(0);
+            service.SendRequest(
+                MakeStorageServiceId(),
+                std::move(resizeRequest));
+            auto resizeResponse = service.RecvResizeFileStoreResponse();
+        }
+
+        auto headers = service.InitSession(fs, "client");
+        ui64 nodeId =
+            service
+                .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file"))
+                ->Record.GetNode()
+                .GetId();
+
+        ui64 handle =
+            service
+                .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR)
+                ->Record.GetHandle();
+
+        auto& runtime = env.GetRuntime();
+
+        size_t throttled = 0;
+        auto writeReadData =
+            [&](ui64 offset, ui64 size, [[maybe_unused]] ui64 expectedFilesize)
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: expectedFilesize надо дропнуть
created_at: 2025-03-21T17:26:56Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
+
+        {
+            auto resizeRequest =
+                service.CreateResizeFileStoreRequest("test", blockCount);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetThrottlingEnabled(true);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxWriteBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxReadBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedWeight(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedTime(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedCount(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetDefaultPostponedRequestWeight(0);
+            service.SendRequest(
+                MakeStorageServiceId(),
+                std::move(resizeRequest));
+            auto resizeResponse = service.RecvResizeFileStoreResponse();
+        }
+
+        auto headers = service.InitSession(fs, "client");
+        ui64 nodeId =
+            service
+                .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file"))
+                ->Record.GetNode()
+                .GetId();
+
+        ui64 handle =
+            service
+                .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR)
+                ->Record.GetHandle();
+
+        auto& runtime = env.GetRuntime();
+
+        size_t throttled = 0;
+        auto writeReadData =
+            [&](ui64 offset, ui64 size, [[maybe_unused]] ui64 expectedFilesize)
+        {
+            auto data = GenerateValidateData(size);
+            {
+                auto request = service.CreateWriteDataRequest(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data);
+                request->CallContext->RequestType =
+                    EFileStoreRequest ::WriteData;
+                service.SendRequest(
+                    MakeStorageServiceId(),
+                    std ::move(request));
+                auto response =
+                    service.RecvResponse<TEvService ::TEvWriteDataResponse>();
+                if (response->GetError().GetCode() == E_FS_THROTTLED) {
+                    ++throttled;
+                }
+            }
+            {
+                auto request = service.CreateReadDataRequest(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data.size());
+                request->CallContext->RequestType =
+                    EFileStoreRequest ::ReadData;
+                service.SendRequest(
+                    MakeStorageServiceId(),
+                    std ::move(request));
+                auto response =
+                    service.RecvResponse<TEvService ::TEvReadDataResponse>();
+                if (response->GetError().GetCode() == E_FS_THROTTLED) {
+                    ++throttled;
+                }
+            }
+        };
+        constexpr auto size =
+            134217728 + 1;   // Config.DefaultThresholds.MaxPostponedWeight
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: 128_MB
created_at: 2025-03-21T17:27:45Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: ShouldThrottleMulipleStageReadsAndWrites / любая другая грамматически корректная форма
created_at: 2025-03-21T17:28:25Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: Сразу надо делать Create с выбранным тобой performance profile: https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/public/api/protos/fs.proto#L153
created_at: 2025-03-21T17:29:22Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
+
+        {
+            auto resizeRequest =
+                service.CreateResizeFileStoreRequest("test", blockCount);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetThrottlingEnabled(true);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxWriteBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxReadBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedWeight(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedTime(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedCount(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetDefaultPostponedRequestWeight(0);
+            service.SendRequest(
+                MakeStorageServiceId(),
+                std::move(resizeRequest));
+            auto resizeResponse = service.RecvResizeFileStoreResponse();
+        }
+
+        auto headers = service.InitSession(fs, "client");
+        ui64 nodeId =
+            service
+                .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file"))
+                ->Record.GetNode()
+                .GetId();
+
+        ui64 handle =
+            service
+                .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR)
+                ->Record.GetHandle();
+
+        auto& runtime = env.GetRuntime();
+
+        size_t throttled = 0;
+        auto writeReadData =
+            [&](ui64 offset, ui64 size, [[maybe_unused]] ui64 expectedFilesize)
+        {
+            auto data = GenerateValidateData(size);
+            {
+                auto request = service.CreateWriteDataRequest(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data);
+                request->CallContext->RequestType =
+                    EFileStoreRequest ::WriteData;
+                service.SendRequest(
+                    MakeStorageServiceId(),
+                    std ::move(request));
+                auto response =
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: Можно сделать 
```c++
service.SendAndRecvWriteData(headers,
                    fs,
                    nodeId,
                    handle,
                    offset,
                    data);
```
created_at: 2025-03-21T17:31:20Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,134 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldMulipleStageThrottle)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+        config.SetSSDThrottlingEnabled(true);
+        config.SetSSDMaxReadBandwidth(maxBandwidth);
+        config.SetSSDMaxWriteBandwidth(maxBandwidth);
+        config.SetSSDMaxPostponedWeight(0);
+        config.SetSSDMaxPostponedTime(0);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        const auto blockCount = 1'000;
+        service.CreateFileStore(fs, blockCount);
+
+        {
+            auto resizeRequest =
+                service.CreateResizeFileStoreRequest("test", blockCount);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetThrottlingEnabled(true);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxWriteBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxReadBandwidth(maxBandwidth);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedWeight(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedTime(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetMaxPostponedCount(0);
+            resizeRequest->Record.MutablePerformanceProfile()
+                ->SetDefaultPostponedRequestWeight(0);
+            service.SendRequest(
+                MakeStorageServiceId(),
+                std::move(resizeRequest));
+            auto resizeResponse = service.RecvResizeFileStoreResponse();
+        }
+
+        auto headers = service.InitSession(fs, "client");
+        ui64 nodeId =
+            service
+                .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file"))
+                ->Record.GetNode()
+                .GetId();
+
+        ui64 handle =
+            service
+                .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR)
+                ->Record.GetHandle();
+
+        auto& runtime = env.GetRuntime();
+
+        size_t throttled = 0;
+        auto writeReadData =
+            [&](ui64 offset, ui64 size, [[maybe_unused]] ui64 expectedFilesize)
+        {
+            auto data = GenerateValidateData(size);
+            {
+                auto request = service.CreateWriteDataRequest(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data);
+                request->CallContext->RequestType =
+                    EFileStoreRequest ::WriteData;
+                service.SendRequest(
+                    MakeStorageServiceId(),
+                    std ::move(request));
+                auto response =
+                    service.RecvResponse<TEvService ::TEvWriteDataResponse>();
+                if (response->GetError().GetCode() == E_FS_THROTTLED) {
+                    ++throttled;
+                }
+            }
+            {
+                auto request = service.CreateReadDataRequest(
+                    headers,
+                    fs,
+                    nodeId,
+                    handle,
+                    offset,
+                    data.size());
+                request->CallContext->RequestType =
+                    EFileStoreRequest ::ReadData;
+                service.SendRequest(
+                    MakeStorageServiceId(),
+                    std ::move(request));
+                auto response =
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: same
created_at: 2025-03-21T17:31:44Z
author_association: COLLABORATOR
diff_hunk: @@ -2776,6 +2775,105 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         // clang-format on
     }
 
+    Y_UNIT_TEST(ShouldThrottleMulipleStageReadsAndWrites)
+    {
+        const auto maxBandwidth = 10000;
+        NProto::TStorageConfig config;
+        config.SetThrottlingEnabled(true);
+        config.SetThreeStageWriteEnabled(true);
+        config.SetTwoStageReadEnabled(true);
+        config.SetUnalignedThreeStageWriteEnabled(true);
+        config.SetMultipleStageRequestThrottlingEnabled(true);
+
+        TTestEnv env({}, config);
+        env.CreateSubDomain("nfs");
+
+        ui32 nodeIdx = env.CreateNode("nfs");
+
+        TServiceClient service(env.GetRuntime(), nodeIdx);
+        const TString fs = "test";
+        {
+            const auto blockCount = 1'000;
+            auto request = service.CreateCreateFileStoreRequest(fs, blockCount);
+            {
+                auto performanceProfile =
+                    request->Record.MutablePerformanceProfile();
+                performanceProfile->SetThrottlingEnabled(true);
+                performanceProfile->SetMaxWriteBandwidth(maxBandwidth);
+                performanceProfile->SetMaxReadBandwidth(maxBandwidth);
+                performanceProfile->SetMaxPostponedWeight(1);
+                performanceProfile->SetMaxPostponedTime(1);
+                performanceProfile->SetMaxPostponedCount(1);
+                performanceProfile->SetDefaultPostponedRequestWeight(1);
+            }
+
+            auto resp = service.SendAndRecvCreateFileStore(std::move(request));
path: cloud/filestore/libs/storage/service/service_ut.cpp
user: login: debnatkh
body: resp – Unused. 
created_at: 2025-03-21T23:32:47Z
author_association: COLLABORATOR

PR comments:
user: login: debnatkh
created_at: 2025-03-12T22:39:24Z
body: - То же нужно в ReadData
- И нужен юниттест `ShouldNotFallback...` в `service_ut`
author_association: COLLABORATOR
user: login: github-actions[bot]
created_at: 2025-03-13T00:09:36Z
body: <!-- status pr=3180, run=9878, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 12750b1cc6b5198ba47d5eb226c6e5799d87110f.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6509](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13823102224/1/nebius-x86-64/summary/ya-test.html) | [6509](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13823102224/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T17:53:30Z
body: <!-- status pr=3180, run=10024, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 5e00cd90e32de951c71b7deff5f2c5433414c143.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6233](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13928259301/1/nebius-x86-64/summary/ya-test.html) | [6231](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13928259301/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [2](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13928259301/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-20T01:59:43Z
body: <!-- status pr=3180, run=10094, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 69917d544de0a36c3ee8f785ddede707f46f1f4b.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6530](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13959590319/1/nebius-x86-64/summary/ya-test.html) | [6530](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13959590319/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-20T22:53:55Z
body: <!-- status pr=3180, run=10136, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 2bde7bb4b35fd5d3ec228e218d2ceb62846baeb4.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6398](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13979872891/1/nebius-x86-64/summary/ya-test.html) | [6377](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13979872891/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13979872891/1/nebius-x86-64/summary/ya-test.html#FAIL) | [16](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13979872891/1/nebius-x86-64/summary/ya-test.html#SKIP) | [4](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13979872891/1/nebius-x86-64/summary/ya-test.html#MUTE) |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-21T12:36:53Z
body: <!-- status pr=3180, run=10152, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit f52c205c24c55eedf9fa081c1d14ea6a6147ce5e.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6533](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13990696264/1/nebius-x86-64/summary/ya-test.html) | [6533](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13990696264/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-21T20:58:17Z
body: <!-- status pr=3180, run=10164, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit dbd088ea1ee2729bb83db4a4bec26022f9d2f0e3.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6535](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13999544143/1/nebius-x86-64/summary/ya-test.html) | [6519](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13999544143/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | [12](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13999544143/1/nebius-x86-64/summary/ya-test.html#SKIP) | [4](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13999544143/1/nebius-x86-64/summary/ya-test.html#MUTE) |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-21T23:40:43Z
body: <!-- status pr=3180, run=10167, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit f59e7f46fb0742b55e16a67c38a560633f0f3a4d.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6534](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14001912417/1/nebius-x86-64/summary/ya-test.html) | [6527](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14001912417/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | [5](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14001912417/1/nebius-x86-64/summary/ya-test.html#SKIP) | [2](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14001912417/1/nebius-x86-64/summary/ya-test.html#MUTE) |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-22T03:07:41Z
body: <!-- status pr=3180, run=10168, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 430d414617a4429673482f9bca9dde8c0dcd1266.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [6533](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14004010509/1/nebius-x86-64/summary/ya-test.html) | [6533](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14004010509/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR