Repository: ydb-platform/nbs
Pull request #3187: issue-3: add device timeout notify to rdma partition
Author: login: vladstepanyuk
Labels: name: large-tests
description: Launch large tests for PR
name: blockstore
description: Add this label to run only cloud/blockstore build and tests on PR
Created: 2025-03-14T09:58:14Z | Merged: 2025-04-24T04:59:48Z

#3 
Adding volume notifying if some of requests time outed or rdma endpoint unavailable

В отдельном пре уменьшу колличесвто бойлер плейт кода в рдма ревест хэндлерах. в этом только добавляю отправку нотификации о  таймаутах запросов в девайсы


Diff:
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl.cpp
index 6d2180069f4..d3c0c389ac5 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl.cpp
@@ -1,6 +1,7 @@
 #include "part_nonrepl.h"
 
 #include "part_nonrepl_actor.h"
+#include "part_nonrepl_rdma.h"
 #include "part_nonrepl_rdma_actor.h"
 
 namespace NCloud::NBlockStore::NStorage {
@@ -17,7 +18,7 @@ IActorPtr CreateNonreplicatedPartition(
     NRdma::IClientPtr rdmaClient)
 {
     if (rdmaClient) {
-        return std::make_unique<TNonreplicatedPartitionRdmaActor>(
+        return CreateNonreplicatedPartitionRdma(
             std::move(config),
             std::move(diagnosticsConfig),
             std::move(partConfig),
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h
index 08b6f8edc6a..04eb8b482fe 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h
@@ -217,9 +217,12 @@ struct TEvNonreplPartitionPrivate
         // Request execution total time.
         TDuration ExecutionTime;
 
-        // Indexes of devices that participated in the request.
+        // Indices of devices that participated in the request.
         TStackVec<ui32, 2> DeviceIndices;
 
+        // Indices of devices where requests have resulted in errors.
+        TStackVec<ui32, 2> ErrorDeviceIndices;
+
         ui32 NonVoidBlockCount = 0;
         ui32 VoidBlockCount = 0;
     };
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
index 32c3cfb28db..3438c617784 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
@@ -22,6 +22,13 @@ LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER);
 
 ////////////////////////////////////////////////////////////////////////////////
 
+bool NeedToNotifyAboutDeviceRequestError(const NProto::TError& err)
+{
+    return err.GetCode() == E_RDMA_UNAVAILABLE || err.GetCode() == E_TIMEOUT;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 TNonreplicatedPartitionRdmaActor::TNonreplicatedPartitionRdmaActor(
         TStorageConfigPtr config,
         TDiagnosticsConfigPtr diagnosticsConfig,
@@ -251,6 +258,7 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
         ui64 sz = r.DeviceBlockRange.Size() * PartConfig->GetBlockSize();
         dr->StartIndexOffset = startBlockIndexOffset;
         dr->BlockCount = r.DeviceBlockRange.Size();
+        dr->DeviceIdx = r.DeviceIdx;
         startBlockIndexOffset += r.DeviceBlockRange.Size();
 
         NProto::TReadDeviceBlocksRequest deviceRequest;
@@ -300,6 +308,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (inserted) {
+        timedOutDeviceCtx.FirstErrorTs = ctx.Now();
+        timedOutDeviceCtx.VolumeWasNotified = false;
+        return;
+    }
+
+    if (timedOutDeviceCtx.VolumeWasNotified) {
+        return;
+    }
+
+    auto timePassedSinceFirstError = ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+    if (timePassedSinceFirstError >= Config->GetLaggingDeviceTimeoutThreshold())
+    {
+        NCloud::Send(
+            ctx,
+            PartConfig->GetParentActorId(),
+            std::make_unique<TEvVolumePrivate::TEvDeviceTimedOutRequest>(
+                deviceUUID));
+        timedOutDeviceCtx.VolumeWasNotified = true;
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompleted(
+    const NActors::TActorContext& ctx,
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    const auto& devices = PartConfig->GetDevices();
+    for (auto idx: opCompleted.DeviceIndices) {
+        Y_ABORT_UNLESS(idx < static_cast<ui32>(devices.size()));
+        const auto* errDevice = FindPtr(opCompleted.ErrorDeviceIndices, idx);
+        const auto& deviceUUID = devices[idx].GetDeviceUUID();
+        if (errDevice) {
+            NotifyDeviceTimedOutIfNeeded(ctx, deviceUUID);
+            continue;
+        }
+
+        TimedOutDeviceCtxByDeviceUUID.erase(deviceUUID);
+    }
+}
+
 void TNonreplicatedPartitionRdmaActor::SendRdmaUnavailableIfNeeded(
     const TActorContext& ctx)
 {
@@ -310,8 +370,7 @@ void TNonreplicatedPartitionRdmaActor::SendRdmaUnavailableIfNeeded(
     NCloud::Send(
         ctx,
         PartConfig->GetParentActorId(),
-        std::make_unique<TEvVolume::TEvRdmaUnavailable>()
-    );
+        std::make_unique<TEvVolume::TEvRdmaUnavailable>());
 
     ReportRdmaError("RDMA Unavailable");
 
@@ -331,6 +390,8 @@ void TNonreplicatedPartitionRdmaActor::HandleReadBlocksCompleted(
 
     UpdateStats(msg->Stats);
 
+    ProcessOperationCompleted(ctx, *msg);
+
     const auto requestBytes = msg->Stats.GetUserReadCounters().GetBlocksCount()
         * PartConfig->GetBlockSize();
     const auto time = CyclesToDurationSafe(msg->TotalCycles).MicroSeconds();
@@ -370,6 +431,8 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksCompleted(
 
     UpdateStats(msg->Stats);
 
+    ProcessOperationCompleted(ctx, *msg);
+
     const auto requestBytes = msg->Stats.GetUserWriteCounters().GetBlocksCount()
         * PartConfig->GetBlockSize();
     const auto time = CyclesToDurationSafe(msg->TotalCycles).MicroSeconds();
@@ -399,6 +462,8 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocksCompleted(
 
     UpdateStats(msg->Stats);
 
+    ProcessOperationCompleted(ctx, *msg);
+
     const auto requestBytes = msg->Stats.GetUserWriteCounters().GetBlocksCount()
         * PartConfig->GetBlockSize();
     const auto time = CyclesToDurationSafe(msg->TotalCycles).MicroSeconds();
@@ -425,6 +490,8 @@ void TNonreplicatedPartitionRdmaActor::HandleChecksumBlocksCompleted(
 
     UpdateStats(msg->Stats);
 
+    ProcessOperationCompleted(ctx, *msg);
+
     const auto requestBytes = msg->Stats.GetSysChecksumCounters().GetBlocksCount()
         * PartConfig->GetBlockSize();
     const auto time = CyclesToDurationSafe(msg->TotalCycles).MicroSeconds();
@@ -511,6 +578,37 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable (lagging)",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsBackOnline(
+    const TEvNonreplPartitionPrivate::TEvAgentIsBackOnline::TPtr& ev,
+    const TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    const auto& agentId = msg->AgentId;
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Lagging agent %s is back online",
+        PartConfig->GetName().c_str(),
+        agentId.Quote().c_str());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 bool TNonreplicatedPartitionRdmaActor::HandleRequests(STFUNC_SIG)
 {
     switch (ev->GetTypeRewrite()) {
@@ -593,6 +691,14 @@ STFUNC(TNonreplicatedPartitionRdmaActor::StateWork)
 
         IgnoreFunc(TEvVolume::TEvRWClientIdChanged);
 
+        HFunc(
+            TEvNonreplPartitionPrivate::TEvAgentIsBackOnline,
+            HandleAgentIsBackOnline);
+        HFunc(
+            TEvNonreplPartitionPrivate::TEvAgentIsUnavailable,
+            HandleAgentIsUnavailable);
+        IgnoreFunc(TEvVolumePrivate::TEvDeviceTimedOutResponse);
+
         default:
             if (!HandleRequests(ev)) {
                 HandleUnexpectedEvent(ev, TBlockStoreComponents::PARTITION);
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
index c50393e2203..1079a9d78c8 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
@@ -15,6 +15,7 @@
 #include <cloud/blockstore/libs/storage/model/requests_in_progress.h>
 #include <cloud/blockstore/libs/storage/partition_common/drain_actor_companion.h>
 #include <cloud/blockstore/libs/storage/partition_common/get_device_for_range_companion.h>
+#include <cloud/blockstore/libs/storage/volume/volume_events_private.h>
 
 #include <contrib/ydb/library/actors/core/actor_bootstrapped.h>
 #include <contrib/ydb/library/actors/core/events.h>
@@ -27,11 +28,19 @@ namespace NCloud::NBlockStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceReadRequestContext: public NRdma::TNullContext
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    ui32 DeviceIdx = 0;
+};
+
+struct TDeviceReadRequestContext: public TDeviceRequestContext
 {
     ui64 StartIndexOffset = 0;
     ui64 BlockCount = 0;
 };
+////////////////////////////////////////////////////////////////////////////////
+
+bool NeedToNotifyAboutDeviceRequestError(const NProto::TError& err);
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -67,6 +76,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TTimedOutDeviceCtx {
+        TInstant FirstErrorTs;
+        bool VolumeWasNotified = false;
+    };
+    THashMap<TString, TTimedOutDeviceCtx> TimedOutDeviceCtxByDeviceUUID;
     bool SentRdmaUnavailableNotification = false;
 
     const bool AssignIdToWriteAndZeroRequestsEnabled{
@@ -89,6 +103,12 @@ class TNonreplicatedPartitionRdmaActor final
     bool CheckReadWriteBlockRange(const TBlockRange64& range) const;
     void ScheduleCountersUpdate(const NActors::TActorContext& ctx);
     void SendStats(const NActors::TActorContext& ctx);
+    void NotifyDeviceTimedOutIfNeeded(
+        const NActors::TActorContext& ctx,
+        const TString& deviceUUID);
+    void ProcessOperationCompleted(
+        const NActors::TActorContext& ctx,
+        const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted);
     void SendRdmaUnavailableIfNeeded(const NActors::TActorContext& ctx);
     void UpdateStats(const NProto::TPartitionStats& update);
 
@@ -141,6 +161,14 @@ class TNonreplicatedPartitionRdmaActor final
         const TEvNonreplPartitionPrivate::TEvChecksumBlocksCompleted::TPtr& ev,
         const NActors::TActorContext& ctx);
 
+    void HandleAgentIsUnavailable(
+        const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+        const NActors::TActorContext& ctx);
+
+    void HandleAgentIsBackOnline(
+        const TEvNonreplPartitionPrivate::TEvAgentIsBackOnline::TPtr& ev,
+        const NActors::TActorContext& ctx);
+
     bool HandleRequests(STFUNC_SIG);
 
     BLOCKSTORE_IMPLEMENT_REQUEST(ReadBlocks, TEvService);
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
index d5f6389d68a..117d1af93ca 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
@@ -23,7 +23,7 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceChecksumRequestContext: public NRdma::TNullContext
+struct TDeviceChecksumRequestContext: public TDeviceRequestContext
 {
     ui64 RangeStartIndex = 0;
     ui32 RangeSize = 0;
@@ -57,6 +57,12 @@ class TRdmaRequestContext: public NRdma::IClientHandler
     NActors::TActorId ParentActorId;
     ui64 RequestId;
 
+    // Indices of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indices of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
+
 public:
     TRdmaRequestContext(
             TActorSystem* actorSystem,
@@ -75,7 +81,9 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         , RequestId(requestId)
     {}
 
-    void HandleResult(const TDeviceChecksumRequestContext& dc, TStringBuf buffer)
+    void HandleResult(
+        const TDeviceChecksumRequestContext& reqCtx,
+        TStringBuf buffer)
     {
         auto* serializer = TBlockStoreProtocol::Serializer();
         auto [result, err] = serializer->Parse(buffer);
@@ -92,9 +100,9 @@ class TRdmaRequestContext: public NRdma::IClientHandler
             return;
         }
 
-        Checksums[dc.RangeStartIndex] = {
+        Checksums[reqCtx.RangeStartIndex] = {
             concreteProto.GetChecksum(),
-            dc.RangeSize};
+            reqCtx.RangeSize};
     }
 
     void HandleResponse(
@@ -106,14 +114,19 @@ class TRdmaRequestContext: public NRdma::IClientHandler
 
         auto guard = Guard(Lock);
 
-        auto* dc =
+        auto* reqCtx =
             static_cast<TDeviceChecksumRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dc, buffer);
+            HandleResult(*reqCtx, buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
 
         if (--ResponseCount != 0) {
@@ -144,6 +157,8 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
         auto& counters = *completion->Stats.MutableSysChecksumCounters();
         completion->TotalCycles = RequestInfo->GetTotalCycles();
+        completion->DeviceIndices = DeviceIndices;
+        completion->ErrorDeviceIndices = ErrorDeviceIndices;
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
@@ -225,6 +240,7 @@ void TNonreplicatedPartitionRdmaActor::HandleChecksumBlocks(
         auto dc = std::make_unique<TDeviceChecksumRequestContext>();
         dc->RangeStartIndex = r.BlockRange.Start;
         dc->RangeSize = r.DeviceBlockRange.Size() * PartConfig->GetBlockSize();
+        dc->DeviceIdx = r.DeviceIdx;
 
         NProto::TChecksumDeviceBlocksRequest deviceRequest;
         deviceRequest.MutableHeaders()->CopyFrom(msg->Record.GetHeaders());
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
index 892e71e50bc..aeaacc4a553 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
@@ -38,9 +38,16 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
     TAdaptiveLock Lock;
     size_t ResponseCount;
     NProto::TReadBlocksResponse Response;
+    NProto::TError Error;
 
     ui32 VoidBlockCount = 0;
 
+    // Indices of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indices of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
+
 public:
     TRdmaRequestReadBlocksContext(
             TActorSystem* actorSystem,
@@ -68,20 +75,22 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         }
     }
 
-    void HandleResult(const TDeviceReadRequestContext& dr, TStringBuf buffer)
+    void HandleResult(
+        const TDeviceReadRequestContext& reqCtx,
+        TStringBuf buffer)
     {
         auto* serializer = TBlockStoreProtocol::Serializer();
         auto [result, err] = serializer->Parse(buffer);
 
         if (HasError(err)) {
-            *Response.MutableError() = std::move(err);
+            Error = std::move(err);
             return;
         }
 
         const auto& concreteProto =
             static_cast<NProto::TReadDeviceBlocksResponse&>(*result.Proto);
         if (HasError(concreteProto.GetError())) {
-            *Response.MutableError() = concreteProto.GetError();
+            Error = concreteProto.GetError();
             return;
         }
 
@@ -91,7 +100,7 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         ui64 b = 0;
         bool isAllZeroes = CheckVoidBlocks;
         while (offset < result.Data.size()) {
-            ui64 targetBlock = dr.StartIndexOffset + b;
+            ui64 targetBlock = reqCtx.StartIndexOffset + b;
             Y_ABORT_UNLESS(targetBlock < static_cast<ui64>(blocks.size()));
             ui64 bytes =
                 Min(result.Data.size() - offset, blocks[targetBlock].size());
@@ -110,7 +119,7 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         }
 
         if (isAllZeroes) {
-            VoidBlockCount += dr.BlockCount;
+            VoidBlockCount += reqCtx.BlockCount;
         }
     }
 
@@ -123,13 +132,19 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
 
         auto guard = Guard(Lock);
 
-        auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
+        auto* reqCtx =
+            static_cast<TDeviceReadRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dr, buffer);
+            HandleResult(*reqCtx, buffer);
         } else {
-            *Response.MutableError() = NRdma::ParseError(buffer);
+            Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
 
         if (--ResponseCount != 0) {
@@ -138,7 +153,8 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
 
         // Got all device responses. Do processing.
 
-        ProcessError(*ActorSystem, *PartConfig, *Response.MutableError());
+        ProcessError(*ActorSystem, *PartConfig, Error);
+        *Response.MutableError() = Error;
         auto error = Response.GetError();
 
         const ui32 blockCount = Response.GetBlocks().BuffersSize();
@@ -163,6 +179,8 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         completion->TotalCycles = RequestInfo->GetTotalCycles();
         completion->NonVoidBlockCount = allZeroes ? 0 : blockCount;
         completion->VoidBlockCount = allZeroes ? blockCount : 0;
+        completion->DeviceIndices = DeviceIndices;
+        completion->ErrorDeviceIndices = ErrorDeviceIndices;
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
index 29ab6dbad16..c431dd01231 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
@@ -41,6 +41,12 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
 
     ui32 VoidBlockCount = 0;
 
+    // Indices of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indices of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
+
 public:
     TRdmaRequestReadBlocksLocalContext(
             TActorSystem* actorSystem,
@@ -61,58 +67,62 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
         , CheckVoidBlocks(checkVoidBlocks)
         , ResponseCount(requestCount)
         , SgList(std::move(sglist))
-    {
-    }
+    {}
 
-    void HandleResult(const TDeviceReadRequestContext& dr, TStringBuf buffer)
+    void HandleResult(
+        const TDeviceReadRequestContext& reqCtx,
+        TStringBuf buffer)
     {
-        if (auto guard = SgList.Acquire()) {
-            auto* serializer = TBlockStoreProtocol::Serializer();
-            auto [result, err] = serializer->Parse(buffer);
-
-            if (HasError(err)) {
-                Error = std::move(err);
-                return;
-            }
-
-            const auto& concreteProto =
-                static_cast<NProto::TReadDeviceBlocksResponse&>(*result.Proto);
-            if (HasError(concreteProto.GetError())) {
-                Error = concreteProto.GetError();
-                return;
-            }
+        auto guard = SgList.Acquire();
+        if (!guard) {
+            Error = MakeError(E_CANCELLED, "can't acquire sglist");
+            return;
+        }
+        auto* serializer = TBlockStoreProtocol::Serializer();
+        auto [result, err] = serializer->Parse(buffer);
 
-            TSgList data = guard.Get();
+        if (HasError(err)) {
+            Error = std::move(err);
+            return;
+        }
 
-            ui64 offset = 0;
-            ui64 b = 0;
-            bool isAllZeroes = CheckVoidBlocks;
-            while (offset < result.Data.size()) {
-                ui64 targetBlock = dr.StartIndexOffset + b;
-                Y_ABORT_UNLESS(targetBlock < data.size());
-                ui64 bytes =
-                    Min(result.Data.size() - offset, data[targetBlock].Size());
-                Y_ABORT_UNLESS(bytes);
+        const auto& concreteProto =
+            static_cast<NProto::TReadDeviceBlocksResponse&>(*result.Proto);
+        if (HasError(concreteProto.GetError())) {
+            Error = concreteProto.GetError();
+            return;
+        }
 
-                char* dst = const_cast<char*>(data[targetBlock].Data());
-                const char* src = result.Data.data() + offset;
+        TSgList data = guard.Get();
 
-                if (isAllZeroes) {
-                    isAllZeroes = IsAllZeroes(src, bytes);
-                }
+        ui64 offset = 0;
+        ui64 b = 0;
+        bool isAllZeroes = CheckVoidBlocks;
+        while (offset < result.Data.size()) {
+            ui64 targetBlock = reqCtx.StartIndexOffset + b;
+            Y_ABORT_UNLESS(targetBlock < data.size());
+            ui64 bytes =
+                Min(result.Data.size() - offset, data[targetBlock].Size());
+            Y_ABORT_UNLESS(bytes);
 
-                // may be nullptr for overlay disks
-                if (dst) {
-                    memcpy(dst, src, bytes);
-                }
+            char* dst = const_cast<char*>(data[targetBlock].Data());
+            const char* src = result.Data.data() + offset;
 
-                offset += bytes;
-                ++b;
+            if (isAllZeroes) {
+                isAllZeroes = IsAllZeroes(src, bytes);
             }
 
-            if (isAllZeroes) {
-                VoidBlockCount += dr.BlockCount;
+            // may be nullptr for overlay disks
+            if (dst) {
+                memcpy(dst, src, bytes);
             }
+
+            offset += bytes;
+            ++b;
+        }
+
+        if (isAllZeroes) {
+            VoidBlockCount += reqCtx.BlockCount;
         }
     }
 
@@ -125,13 +135,19 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
 
         auto guard = Guard(Lock);
 
-        auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
+        auto* reqCtx =
+            static_cast<TDeviceReadRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dr, buffer);
+            HandleResult(*reqCtx, buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
 
         if (--ResponseCount != 0) {
@@ -159,6 +175,8 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
         auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
         auto& counters = *completion->Stats.MutableUserReadCounters();
         completion->TotalCycles = RequestInfo->GetTotalCycles();
+        completion->DeviceIndices = DeviceIndices;
+        completion->ErrorDeviceIndices = ErrorDeviceIndices;
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
index 01d6af3531f..9e36a2eaced 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
@@ -45,6 +45,12 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
     const NActors::TActorId ParentActorId;
     const ui64 RequestId;
 
+    // Indices of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indices of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
+
 public:
     TRdmaWriteBlocksResponseHandler(
             TActorSystem* actorSystem,
@@ -102,11 +108,17 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* reqCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
+
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
 
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
 
         if (--ResponseCount != 0) {
@@ -131,6 +143,8 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
         auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
         auto& counters = *completion->Stats.MutableUserWriteCounters();
         completion->TotalCycles = RequestInfo->GetTotalCycles();
+        completion->DeviceIndices = DeviceIndices;
+        completion->ErrorDeviceIndices = ErrorDeviceIndices;
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
@@ -253,9 +267,12 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks(
             deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1);
         }
 
+        auto context = std::make_unique<TDeviceRequestContext>();
+        context->DeviceIdx = r.DeviceIdx;
+
         auto [req, err] = ep->AllocateRequest(
             requestResponseHandler,
-            nullptr,
+            std::move(context),
             NRdma::TProtoMessageSerializer::MessageByteSize(
                 deviceRequest,
                 r.DeviceBlockRange.Size() * PartConfig->GetBlockSize()),
@@ -400,10 +417,12 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal(
                 msg->Record.GetHeaders().GetVolumeRequestId());
             deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1);
         }
+        auto context = std::make_unique<TDeviceRequestContext>();
+        context->DeviceIdx = r.DeviceIdx;
 
         auto [req, err] = ep->AllocateRequest(
             requestResponseHandler,
-            nullptr,
+            std::move(context),
             NRdma::TProtoMessageSerializer::MessageByteSize(
                 deviceRequest,
                 r.DeviceBlockRange.Size() * PartConfig->GetBlockSize()),
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
index 21ac2ab8c7e..0491756291f 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
@@ -38,6 +38,12 @@ class TRdmaRequestContext: public NRdma::IClientHandler
     NActors::TActorId ParentActorId;
     ui64 RequestId;
 
+    // Indices of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indices of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
+
 public:
     TRdmaRequestContext(
             TActorSystem* actorSystem,
@@ -83,11 +89,17 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* reqCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
+
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
 
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
 
         if (--ResponseCount != 0) {
@@ -112,6 +124,8 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
         auto& counters = *completion->Stats.MutableUserWriteCounters();
         completion->TotalCycles = RequestInfo->GetTotalCycles();
+        completion->DeviceIndices = DeviceIndices;
+        completion->ErrorDeviceIndices = ErrorDeviceIndices;
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
@@ -201,9 +215,12 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocks(
             deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1);
         }
 
+        auto context = std::make_unique<TDeviceRequestContext>();
+        context->DeviceIdx = r.DeviceIdx;
+
         auto [req, err] = ep->AllocateRequest(
             requestContext,
-            nullptr,
+            std::move(context),
             NRdma::TProtoMessageSerializer::MessageByteSize(deviceRequest, 0),
             4_KB);
 
diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
index 7bf1fc20af8..e198c1a6a5c 100644
--- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
+++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
@@ -28,6 +28,8 @@ namespace NCloud::NBlockStore::NStorage {
 using namespace NActors;
 using namespace NKikimr;
 
+using namespace std::chrono_literals;
+
 namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -40,6 +42,7 @@ struct TTestEnv
     TStorageStatsServiceStatePtr StorageStatsServiceState;
     TDiskAgentStatePtr DiskAgentState;
     NRdma::IClientPtr RdmaClient;
+    TStorageConfigPtr Config;
 
     static void AddDevice(
         ui32 nodeId,
@@ -85,7 +88,8 @@ struct TTestEnv
             TTestActorRuntime& runtime,
             NProto::EVolumeIOMode ioMode,
             TDevices devices,
-            bool optimizeVoidBuffersTransfer)
+            bool optimizeVoidBuffersTransfer,
+            bool laggingDevicesAllowed = false)
         : Runtime(runtime)
         , ActorId(0, "YYY")
         , VolumeActorId(0, "VVV")
@@ -109,6 +113,8 @@ struct TTestEnv
                 NCloud::NProto::TFeaturesConfig())
         );
 
+        Config = config;
+
         auto nodeId = Runtime.GetNodeId(0);
 
         Runtime.AddLocalService(
@@ -132,6 +138,7 @@ struct TTestEnv
                 VolumeActorId};
         params.IOMode = ioMode;
         params.UseSimpleMigrationBandwidthLimiter = false;
+        params.LaggingDevicesAllowed = laggingDevicesAllowed;
         auto partConfig =
             std::make_shared<TNonreplicatedPartitionConfig>(std::move(params));
 
@@ -571,6 +578,147 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest)
         UNIT_ASSERT_VALUES_EQUAL(1, notificationCount);
     }
 
+    Y_UNIT_TEST(ShouldHandleRequestSendFailure)
+    {
+        TTestBasicRuntime runtime;
+
+        TTestEnv env(
+            runtime,
+            NProto::VOLUME_IO_OK,
+            TTestEnv::DefaultDevices(runtime.GetNodeId(0)),
+            false,
+            true);
+        TPartitionClient client(runtime, env.ActorId);
+
+        TActorId notifiedActor;
+        ui32 deviceTimedOutCount = 0;
+        THashSet<TString> devices;
+
+        runtime.SetObserverFunc(
+            [&](TAutoPtr<IEventHandle>& event)
+            {
+                switch (event->GetTypeRewrite()) {
+                    case TEvVolumePrivate::EvDeviceTimedOutRequest: {
+                        if (event->Recipient != env.VolumeActorId) {
+                            break;
+                        }
+                        notifiedActor = event->Recipient;
+                        auto* ev = static_cast<
+                            TEvVolumePrivate::TEvDeviceTimedOutRequest*>(
+                            event->GetBase());
+                        devices.emplace(ev->DeviceUUID);
+                        ++deviceTimedOutCount;
+                    }
+                }
+
+                return TTestActorRuntime::DefaultObserverFunc(event);
+            });
+
+        env.Rdma().InjectErrors(
+            {},
+            MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable"),
+            {});
+        env.Rdma().InitAllEndpoints();
+        const auto error = MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable");
+
+        const auto blockRange1 = TBlockRange64::WithLength(1024, 3072);
+
+        WRITE_BLOCKS_E(error);
+        READ_BLOCKS_E(error, 0);
+        UNIT_ASSERT_VALUES_EQUAL(0, devices.size());
+        runtime.DispatchEvents({}, 10ms);
+        runtime.AdvanceCurrentTime(
+            env.Config->GetLaggingDeviceTimeoutThreshold() + 1ms);
+        ZERO_BLOCKS_E(error);
+        runtime.DispatchEvents({}, 10ms);
+
+        THashSet<TString> expected = {"vasya", "petya"};
+        UNIT_ASSERT_VALUES_EQUAL(expected.size(), devices.size());
+        for (const auto& d: devices) {
+            UNIT_ASSERT(expected.contains(d));
+        }
+
+        UNIT_ASSERT_VALUES_EQUAL(2, deviceTimedOutCount);
+    }
+
+    Y_UNIT_TEST(ShouldResetDeviceTimeoutInfoOnSucceededRequest)
+    {
+        TTestBasicRuntime runtime;
+
+        TTestEnv env(
+            runtime,
+            NProto::VOLUME_IO_OK,
+            TTestEnv::DefaultDevices(runtime.GetNodeId(0)),
+            false,
+            true);
+        TPartitionClient client(runtime, env.ActorId);
+
+        TActorId notifiedActor;
+        ui32 deviceTimedOutCount = 0;
+        THashSet<TString> devices;
+
+        runtime.SetObserverFunc(
+            [&](TAutoPtr<IEventHandle>& event)
+            {
+                switch (event->GetTypeRewrite()) {
+                    case TEvVolumePrivate::EvDeviceTimedOutRequest: {
+                        if (event->Recipient != env.VolumeActorId) {
+                            break;
+                        }
+                        notifiedActor = event->Recipient;
+                        auto* ev = static_cast<
+                            TEvVolumePrivate::TEvDeviceTimedOutRequest*>(
+                            event->GetBase());
+                        devices.emplace(ev->DeviceUUID);
+                        ++deviceTimedOutCount;
+                    }
+                }
+
+                return TTestActorRuntime::DefaultObserverFunc(event);
+            });
+
+        env.Rdma().InjectErrors(
+            {},
+            MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable"),
+            {});
+        env.Rdma().InitAllEndpoints();
+        const auto error = MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable");
+
+        const auto blockRange1 = TBlockRange64::WithLength(1024, 3072);
+
+        WRITE_BLOCKS_E(error);
+        runtime.DispatchEvents({}, 10ms);
+        UNIT_ASSERT_VALUES_EQUAL(0, devices.size());
+        runtime.AdvanceCurrentTime(
+            env.Config->GetLaggingDeviceTimeoutThreshold() / 2);
+
+        env.Rdma().InjectErrors({}, {}, {});
+        client.ZeroBlocks(blockRange1);
+
+        env.Rdma().InjectErrors(
+            {},
+            MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable"),
+            {});
+        runtime.AdvanceCurrentTime(
+            env.Config->GetLaggingDeviceTimeoutThreshold() / 2 + 1ms);
+        WRITE_BLOCKS_E(error);
+        runtime.DispatchEvents({}, 10ms);
+        UNIT_ASSERT_VALUES_EQUAL(0, devices.size());
+
+        runtime.AdvanceCurrentTime(
+            env.Config->GetLaggingDeviceTimeoutThreshold() + 1ms);
+        WRITE_BLOCKS_E(error);
+        runtime.DispatchEvents({}, 10ms);
+
+        THashSet<TString> expected = {"vasya", "petya"};
+        UNIT_ASSERT_VALUES_EQUAL(expected.size(), devices.size());
+        for (const auto& d: devices) {
+            UNIT_ASSERT(expected.contains(d));
+        }
+
+        UNIT_ASSERT_VALUES_EQUAL(2, deviceTimedOutCount);
+    }
+
     Y_UNIT_TEST(ShouldUpdateStats)
     {
         TTestBasicRuntime runtime;
diff --git a/cloud/blockstore/libs/storage/volume/volume_events_private.h b/cloud/blockstore/libs/storage/volume/volume_events_private.h
index 416d0cee7e0..523dab3030c 100644
--- a/cloud/blockstore/libs/storage/volume/volume_events_private.h
+++ b/cloud/blockstore/libs/storage/volume/volume_events_private.h
@@ -141,7 +141,7 @@ struct TEvVolumePrivate
 
     struct TDeviceTimedOutRequest
     {
-        const TString DeviceUUID;
+        TString DeviceUUID;
 
         explicit TDeviceTimedOutRequest(TString deviceUUID)
             : DeviceUUID(std::move(deviceUUID))


Reviews:
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T10:03:27Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T10:05:00Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T10:05:25Z
user: login: komarevtsev-d
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T21:19:43Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T12:18:04Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T12:19:21Z
user: login: komarevtsev-d
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T19:11:02Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T22:51:20Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T22:52:13Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T22:52:41Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T22:55:39Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:04:44Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:10:56Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:12:21Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:13:00Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:14:20Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:15:34Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:22:47Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:23:15Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:23:41Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:24:38Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:25:33Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:27:52Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:29:32Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:31:00Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:33:15Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:33:36Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:34:09Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-17T23:38:45Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:07:22Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:07:31Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:12:31Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:12:45Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:37:07Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:39:42Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T10:46:15Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T18:00:16Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T18:22:31Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-20T08:56:26Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:58:03Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T11:36:07Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T11:37:30Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T13:44:09Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T22:53:41Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T09:58:56Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T13:11:17Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T10:50:23Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-09T14:51:29Z
user: login: komarevtsev-d
state: DISMISSED
author_association: COLLABORATOR
submitted_at: 2025-04-14T04:21:40Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T10:40:34Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:03:40Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:31:03Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:32:03Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:32:49Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:33:49Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T19:37:07Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T06:09:35Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T14:58:02Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T14:58:09Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T14:58:34Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T14:58:49Z
user: login: SvartMetal
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T14:58:59Z
user: login: SvartMetal
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-23T15:17:17Z
user: login: sharpeye
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-23T19:04:42Z
user: login: komarevtsev-d
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-24T04:37:50Z
user: login: vladstepanyuk
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T04:38:19Z

Review comments:
diff_hunk: @@ -268,6 +270,11 @@ void TNonreplicatedPartitionRdmaActor::HandleChecksumBlocks(
             requestInfo->CallContext);
     }
 
+    TRequestData requestData;
+    for (const auto& r: deviceRequests){
+        requestData.DeviceIndices.emplace_back(r.DeviceIdx);
+    }
+
     RequestsInProgress.AddReadRequest(requestId);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: vladstepanyuk
body: надо передать заполненную requestDat  у в RequestsInProgress
created_at: 2025-03-14T10:03:27Z
author_association: COLLABORATOR
diff_hunk: @@ -83,11 +80,16 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* dr = static_cast<TDeviceRequestContext*>(req->Context.get());
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: vladstepanyuk
body: dCtx
created_at: 2025-03-14T10:05:00Z
author_association: COLLABORATOR
diff_hunk: @@ -132,9 +133,12 @@ struct TTestEnv
                 VolumeActorId};
         params.IOMode = ioMode;
         params.UseSimpleMigrationBandwidthLimiter = false;
+        params.LaggingDevicesAllowed = laggingDevicesAllowed;
         auto partConfig =
             std::make_shared<TNonreplicatedPartitionConfig>(std::move(params));
 
+            // partConfig->
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
user: login: vladstepanyuk
body: убрать
created_at: 2025-03-14T10:05:25Z
author_association: COLLABORATOR
diff_hunk: @@ -3,6 +3,8 @@
 #include "part_nonrepl_actor.h"
 #include "part_nonrepl_rdma_actor.h"
 
+#include <cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma.h>
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl.cpp
user: login: komarevtsev-d
body: ```suggestion
#include "part_nonrepl_rdma.h"
```
created_at: 2025-03-14T20:46:35Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +538,103 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+    auto deviceUUID = std::move(msg->DeviceUUID);
+
+    auto [_, inserted] = DeviceTimeouted.emplace(deviceUUID);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: Кажется стоит удалять при получении TEvAgentIsBackOnline ?
created_at: 2025-03-14T21:09:24Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +538,103 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+    auto deviceUUID = std::move(msg->DeviceUUID);
+
+    auto [_, inserted] = DeviceTimeouted.emplace(deviceUUID);
+    if (!inserted) {
+        return;
+    }
+
+    NCloud::Send(
+        ctx,
+        PartConfig->GetParentActorId(),
+        std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+            std::move(deviceUUID)));
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
+                break;
+            }
+        }
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsBackOnline(
+    const TEvNonreplPartitionPrivate::TEvAgentIsBackOnline::TPtr& ev,
+    const TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    const auto& agentId = msg->AgentId;
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s is back online",
+        PartConfig->GetName().c_str(),
+        agentId.Quote().c_str());
+
+    AgentId2Endpoint.erase(agentId);
+    AgentId2EndpointFuture.erase(agentId);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: В текущей реализации это точно бесполезно, т.к. TEvAgentIsBackOnline придёт только если этот партишн сможет блок прочитать. 

В целом, переделать на то, чтобы `TAgentAvailabilityMonitoringActor` ходил напрямую в диск агента по интерконнекту не сложно и может стоит это сделать.

Но я бы точно не рестартовал эндпоинт если с ним всё норм. Он довольно долго коннектится и это лишние залипания
created_at: 2025-03-14T21:09:47Z
author_association: COLLABORATOR
diff_hunk: @@ -127,7 +124,11 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(*dr, buffer);
         } else {
-            Error = NRdma::ParseError(buffer);
+            auto err = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutError(err)) {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: komarevtsev-d
body: Одна ошибка это слишком мало. Надо как-то заюзать `LaggingDeviceTimeoutThreshold`
created_at: 2025-03-14T21:17:41Z
author_association: COLLABORATOR
diff_hunk: @@ -408,10 +430,12 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal(
                 msg->Record.GetHeaders().GetVolumeRequestId());
             deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1);
         }
+        auto context = std::make_unique<TDeviceRequestContext>();
+        context->DeviceUUID = r.Device.GetDeviceUUID();
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
user: login: vladstepanyuk
body: DeviceIdx не выставил
created_at: 2025-03-17T12:18:04Z
author_association: COLLABORATOR
diff_hunk: @@ -537,7 +544,12 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest)
     {
         TTestBasicRuntime runtime;
 
-        TTestEnv env(runtime);
+        TTestEnv env(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
user: login: vladstepanyuk
body: убрать

created_at: 2025-03-17T12:19:21Z
author_association: COLLABORATOR
diff_hunk: @@ -27,14 +28,39 @@ namespace NCloud::NBlockStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceReadRequestContext: public NRdma::TNullContext
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    TString DeviceUUID;
+    ui32 DeviceIdx;
+};
+
+struct TDeviceReadRequestContext: public TDeviceRequestContext
 {
     ui64 StartIndexOffset = 0;
     ui64 BlockCount = 0;
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 
+class IRdmaDeviceRequestHandler: public NRdma::IClientHandler
+{
+protected:
+    NActors::TActorSystem* ActorSystem;
+    const NActors::TActorId ParentActorId;
+
+public:
+    explicit IRdmaDeviceRequestHandler(
+            NActors::TActorSystem* actorSystem,
+            NActors::TActorId parentActorId)
+        : ActorSystem(actorSystem)
+        , ParentActorId(std::move(parentActorId))
+    {}
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: komarevtsev-d
body: nit: Тут на мой вкус можно добавить как перестраховку
`~IRdmaDeviceRequestHandler() override = default;`

created_at: 2025-03-17T16:57:46Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +111,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TDeviceTimeoutCtx {
+        TInstant FirstErrorTs;
+        bool ParentWasNotified = false;
+    };
+    THashMap<TString, TDeviceTimeoutCtx> DeviceTimeouted;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: komarevtsev-d
body: Может лучше `TimeoutedDevices` ?
created_at: 2025-03-17T17:50:55Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
+                break;
+            }
+        }
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsBackOnline(
+    const TEvNonreplPartitionPrivate::TEvAgentIsBackOnline::TPtr& ev,
+    const TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    const auto& agentId = msg->AgentId;
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s is back online",
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: ```suggestion
        "[%s] Lagging agent %s is back online",
```
Я щас в свои логи везде стараюсь добавлять слово lagging, чтобы дебажить проще было )
created_at: 2025-03-17T18:54:29Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: ```suggestion
        "[%s] Agent %s has become unavailable (lagging)",
```
created_at: 2025-03-17T18:54:43Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: Чтобы не делать два лукапа:
`auto [it, inserted] = DeviceTimeouted.try_emplace(deviceUUID, TDeviceTimeoutCtx{});`
created_at: 2025-03-17T18:56:11Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: В interconnect партишионе есть проблема: таймаут запроса может прийти после того как всё раздуплилось. Тут такое может быть?
created_at: 2025-03-17T18:59:33Z
author_association: COLLABORATOR
diff_hunk: @@ -45,9 +71,27 @@ class TNonreplicatedPartitionRdmaActor final
     const NRdma::IClientPtr RdmaClient;
     const NActors::TActorId StatActorId;
 
-    // TODO implement DeviceStats and similar stuff
-
-    TRequestsInProgress<ui64> RequestsInProgress{EAllowedRequests::ReadWrite};
+    // TODO implement timeouts
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: komarevtsev-d
body: Что за timeouts?
created_at: 2025-03-17T19:00:04Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: А где обработка TEvCancelRequest ?
created_at: 2025-03-17T19:10:21Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: komarevtsev-d
body: Где заполняется поле ActorId?
created_at: 2025-03-17T19:10:39Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +111,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TDeviceTimeoutCtx {
+        TInstant FirstErrorTs;
+        bool ParentWasNotified = false;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: SvartMetal
body: А что такое parent?
created_at: 2025-03-17T22:51:19Z
author_association: COLLABORATOR
diff_hunk: @@ -89,6 +138,11 @@ class TNonreplicatedPartitionRdmaActor final
     bool CheckReadWriteBlockRange(const TBlockRange64& range) const;
     void ScheduleCountersUpdate(const NActors::TActorContext& ctx);
     void SendStats(const NActors::TActorContext& ctx);
+    void NotifyDeviceTimedoutIfNeeded(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: SvartMetal
body: TimedOut
created_at: 2025-03-17T22:52:12Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: timeouted не оч грамотно звучит

DeviceTimedOut
created_at: 2025-03-17T22:52:41Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +111,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TDeviceTimeoutCtx {
+        TInstant FirstErrorTs;
+        bool ParentWasNotified = false;
+    };
+    THashMap<TString, TDeviceTimeoutCtx> DeviceTimeouted;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: SvartMetal
body: DeviceTimeouCtxByDeviceUUID?
created_at: 2025-03-17T22:55:39Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: может отступ?

а то код в кашу слипается
created_at: 2025-03-17T23:04:44Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: Y_ABORT_UNLESS

потому что иначе (в релизе) всё равно грохнется, но с релизной верифайкой будет понятнее что случилось
created_at: 2025-03-17T23:10:56Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: давай уж и на deviceIndex сделаем проверку на границы 
created_at: 2025-03-17T23:12:21Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
+                break;
+            }
+        }
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsBackOnline(
+    const TEvNonreplPartitionPrivate::TEvAgentIsBackOnline::TPtr& ev,
+    const TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    const auto& agentId = msg->AgentId;
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s is back online",
+        PartConfig->GetName().c_str(),
+        agentId.Quote().c_str());
+
+    for (int i = 0; i < PartConfig->GetDevices().size(); ++i) {
+        const auto& device = PartConfig->GetDevices().at(i);
+        if (device.GetAgentId() == msg->AgentId &&
+            DeviceStats[i].DeviceStatus <= EDeviceStatus::Unavailable)
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: и Y_ABORT_UNLESS про принадженость `i` соответствующим границам
created_at: 2025-03-17T23:12:59Z
author_association: COLLABORATOR
diff_hunk: @@ -45,9 +71,27 @@ class TNonreplicatedPartitionRdmaActor final
     const NRdma::IClientPtr RdmaClient;
     const NActors::TActorId StatActorId;
 
-    // TODO implement DeviceStats and similar stuff
-
-    TRequestsInProgress<ui64> RequestsInProgress{EAllowedRequests::ReadWrite};
+    // TODO implement timeouts
+
+    enum class EDeviceStatus
+    {
+        Ok,
+        Unavailable,
+    };
+
+    struct TDeviceStat
+    {
+        EDeviceStatus DeviceStatus = EDeviceStatus::Ok;
+    };
+    TVector<TDeviceStat> DeviceStats;
+
+    struct TRequestData
+    {
+        NActors::TActorId ActorId;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: SvartMetal
body: а какой тут актор имеется ввиду?
created_at: 2025-03-17T23:14:20Z
author_association: COLLABORATOR
diff_hunk: @@ -89,6 +138,11 @@ class TNonreplicatedPartitionRdmaActor final
     bool CheckReadWriteBlockRange(const TBlockRange64& range) const;
     void ScheduleCountersUpdate(const NActors::TActorContext& ctx);
     void SendStats(const NActors::TActorContext& ctx);
+    void NotifyDeviceTimedoutIfNeeded(
+        const NActors::TActorContext& ctx,
+        const TString& deviceUUID);
+    void ProcessOperationCompletedErrors(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: SvartMetal
body: может просто `OperationCompleted`?
created_at: 2025-03-17T23:15:34Z
author_association: COLLABORATOR
diff_hunk: @@ -204,6 +204,7 @@ struct TEvNonreplPartitionPrivate
 
         // Indexes of devices that participated in the request.
         TStackVec<ui32, 2> DeviceIndices;
+        TStackVec<ui32, 2> ErrorDevices;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h
user: login: SvartMetal
body: ErrorDeviceIndices?
created_at: 2025-03-17T23:22:47Z
author_association: COLLABORATOR
diff_hunk: @@ -22,6 +22,26 @@ LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER);
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void IRdmaDeviceRequestHandler::SendDeviceTimedout(TString deviceUUID)
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: SendDeviceTimedOut
created_at: 2025-03-17T23:23:15Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: TimedOut
created_at: 2025-03-17T23:23:41Z
author_association: COLLABORATOR
diff_hunk: @@ -252,6 +252,11 @@ class TNonreplicatedPartitionConfig
         return OutdatedDeviceIds;
     }
 
+    bool GetLaggingDevicesAllowed() const
+    {
+        return LaggingDevicesAllowed;
path: cloud/blockstore/libs/storage/partition_nonrepl/config.h
user: login: SvartMetal
body: какое-то бы разъяснение что это значит...
created_at: 2025-03-17T23:24:38Z
author_association: COLLABORATOR
diff_hunk: @@ -85,7 +88,8 @@ struct TTestEnv
             TTestActorRuntime& runtime,
             NProto::EVolumeIOMode ioMode,
             TDevices devices,
-            bool optimizeVoidBuffersTransfer)
+            bool optimizeVoidBuffersTransfer,
+            bool laggingDevicesAllowed =  false)
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
user: login: SvartMetal
body: лишний пробел перед false
created_at: 2025-03-17T23:25:33Z
author_association: COLLABORATOR
diff_hunk: @@ -252,6 +252,11 @@ class TNonreplicatedPartitionConfig
         return OutdatedDeviceIds;
     }
 
+    bool GetLaggingDevicesAllowed() const
+    {
+        return LaggingDevicesAllowed;
path: cloud/blockstore/libs/storage/partition_nonrepl/config.h
user: login: SvartMetal
body: фича звучит как "остреливай лагающие девайсы"?
created_at: 2025-03-17T23:27:52Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: как-то странно выглядит Canceled в кач-ве reason для Cancel

может тут имеется ввиду EReason::Unavailable?
created_at: 2025-03-17T23:29:32Z
author_association: COLLABORATOR
diff_hunk: @@ -126,10 +125,17 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        AllDevices.emplace_back(dr->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(*dr, buffer);
         } else {
-            *Response.MutableError() = NRdma::ParseError(buffer);
+            auto err = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutError(err)) {
+                ErrDevices.emplace_back(dr->DeviceIdx);
+                SendDeviceTimedout(std::move(dr->DeviceUUID));
+            }
+            *Response.MutableError() = std::move(err);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: SvartMetal
body: ох как у нас много бойлерплейта...


created_at: 2025-03-17T23:31:00Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompletedErrors(
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    for (size_t dIdx: opCompleted.DeviceIndices) {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: dIdx?

тогда уж просто idx
created_at: 2025-03-17T23:33:15Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompletedErrors(
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    for (size_t dIdx: opCompleted.DeviceIndices) {
+        const auto* errDevice = FindPtr(opCompleted.ErrorDevices, dIdx);
+        if (errDevice) {
+            continue;
+        }
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: отступ не повредил бы
created_at: 2025-03-17T23:33:35Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompletedErrors(
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    for (size_t dIdx: opCompleted.DeviceIndices) {
+        const auto* errDevice = FindPtr(opCompleted.ErrorDevices, dIdx);
+        if (errDevice) {
+            continue;
+        }
+        const auto& deviceUUID = PartConfig->GetDevices()[dIdx].GetDeviceUUID();
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: тоже бы Y_ABORT_UNLESS с проверкой границ
created_at: 2025-03-17T23:34:09Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompletedErrors(
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    for (size_t dIdx: opCompleted.DeviceIndices) {
+        const auto* errDevice = FindPtr(opCompleted.ErrorDevices, dIdx);
+        if (errDevice) {
+            continue;
+        }
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: вот эту логику не понял

давай обсудим голосом? 
created_at: 2025-03-17T23:38:45Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: Вынесу всю логику с отменой запросов в отдельный пр

created_at: 2025-03-18T10:07:22Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: Вынесу всю логику с отменой запросов в отдельный  пр
created_at: 2025-03-18T10:07:31Z
author_association: COLLABORATOR
diff_hunk: @@ -45,9 +71,27 @@ class TNonreplicatedPartitionRdmaActor final
     const NRdma::IClientPtr RdmaClient;
     const NActors::TActorId StatActorId;
 
-    // TODO implement DeviceStats and similar stuff
-
-    TRequestsInProgress<ui64> RequestsInProgress{EAllowedRequests::ReadWrite};
+    // TODO implement timeouts
+
+    enum class EDeviceStatus
+    {
+        Ok,
+        Unavailable,
+    };
+
+    struct TDeviceStat
+    {
+        EDeviceStatus DeviceStatus = EDeviceStatus::Ok;
+    };
+    TVector<TDeviceStat> DeviceStats;
+
+    struct TRequestData
+    {
+        NActors::TActorId ActorId;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: vladstepanyuk
body: Вынесу всю логику с отменой запросов в отдельный пр
created_at: 2025-03-18T10:12:31Z
author_association: COLLABORATOR
diff_hunk: @@ -499,6 +575,81 @@ void TNonreplicatedPartitionRdmaActor::HandlePoisonPill(
     ReplyAndDie(ctx);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+void TNonreplicatedPartitionRdmaActor::HandleDeviceTimeoutedRequest(
+    const TEvVolumePrivate::TEvDeviceTimeoutedRequest::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* msg = ev->Get();
+
+    NotifyDeviceTimedoutIfNeeded(ctx, msg->DeviceUUID);
+}
+
+void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable(
+    const TEvNonreplPartitionPrivate::TEvAgentIsUnavailable::TPtr& ev,
+    const NActors::TActorContext& ctx)
+{
+    const auto* msg = ev->Get();
+    LOG_INFO(
+        ctx,
+        TBlockStoreComponents::PARTITION,
+        "[%s] Agent %s has become unavailable",
+        PartConfig->GetName().c_str(),
+        msg->LaggingAgent.GetAgentId().Quote().c_str());
+
+    for (const auto& laggingDevice: msg->LaggingAgent.GetDevices()) {
+        Y_DEBUG_ABORT_UNLESS(DeviceStats.size() > laggingDevice.GetRowIndex());
+        DeviceStats[laggingDevice.GetRowIndex()].DeviceStatus =
+            EDeviceStatus::Unavailable;
+    }
+    using EReason = TEvNonreplPartitionPrivate::TCancelRequest::EReason;
+    for (const auto& [_, requestInfo]: RequestsInProgress.AllRequests()) {
+        for (int deviceIndex: requestInfo.Value.DeviceIndices) {
+            if (PartConfig->GetDevices()[deviceIndex].GetAgentId() ==
+                msg->LaggingAgent.GetAgentId())
+            {
+                NCloud::Send(
+                    ctx,
+                    requestInfo.Value.ActorId,
+                    std::make_unique<
+                        TEvNonreplPartitionPrivate::TEvCancelRequest>(
+                        EReason::Canceled));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: Вынесу всю логику с отменой запросов в отдельный пр
created_at: 2025-03-18T10:12:45Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +111,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TDeviceTimeoutCtx {
+        TInstant FirstErrorTs;
+        bool ParentWasNotified = false;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: vladstepanyuk
body: PartConfig->GetParentActorId()
т.е. волум
created_at: 2025-03-18T10:37:07Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +111,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TDeviceTimeoutCtx {
+        TInstant FirstErrorTs;
+        bool ParentWasNotified = false;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: vladstepanyuk
body: переименовал в VolumeWasNotified
created_at: 2025-03-18T10:39:42Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +317,52 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedoutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto* deviceTimeoutCtx = DeviceTimeouted.FindPtr(deviceUUID);
+    if (deviceTimeoutCtx && deviceTimeoutCtx->ParentWasNotified) {
+        return;
+    }
+
+    if (deviceTimeoutCtx) {
+        auto timePassedSinceFirstError =
+            ctx.Now() - deviceTimeoutCtx->FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            deviceTimeoutCtx->ParentWasNotified = true;
+        }
+        return;
+    }
+
+    auto& dCtx = DeviceTimeouted[deviceUUID];
+    dCtx.FirstErrorTs = ctx.Now();
+    dCtx.ParentWasNotified = false;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: если пришел таймаут то ответ на реквест уже не получим, если ты спрашивал про это.
Здесь https://github.com/ydb-platform/nbs/blob/9a7b1be84ed67df137c45d43e9c9d6c0a91b24b4/cloud/blockstore/libs/rdma/impl/client.cpp#L1529-L1548
мы извлекаем таймаученный запрос отвекчаем на него и соответственно уничтожаем
created_at: 2025-03-18T10:46:14Z
author_association: COLLABORATOR
diff_hunk: @@ -33,6 +53,7 @@ TNonreplicatedPartitionRdmaActor::TNonreplicatedPartitionRdmaActor(
     , PartConfig(std::move(partConfig))
     , RdmaClient(std::move(rdmaClient))
     , StatActorId(statActorId)
+    , DeviceStats(PartConfig->GetDevices().size())
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: убрать
created_at: 2025-03-18T18:00:16Z
author_association: COLLABORATOR
diff_hunk: @@ -83,11 +82,19 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* dCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
+
+        AllDevices.emplace_back(dCtx->DeviceIdx);
 
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(buffer);
         } else {
-            Error = NRdma::ParseError(buffer);
+            auto err = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutError(err)) {
+                ErrDevices.emplace_back(dCtx->DeviceIdx);
+                SendDeviceTimedOut(std::move(dCtx->DeviceUUID));
+            }
+            Error = std::move(err);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: sharpeye
body: Надо что-то придумать от дублирования кода.
created_at: 2025-03-19T18:18:30Z
author_association: COLLABORATOR
diff_hunk: @@ -160,6 +165,12 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
         completion->ExecCycles = RequestInfo->GetExecCycles();
         completion->NonVoidBlockCount = allZeroes ? 0 : RequestBlockCount;
         completion->VoidBlockCount = allZeroes ? RequestBlockCount : 0;
+        std::ranges::copy(
+            ErrDevices,
+            std::back_inserter(completion->ErrorDeviceIndices));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: sharpeye
body: ```cpp
ErrDevices = completion->ErrorDeviceIndices;
AllDevices = completion->DeviceIndices;
```
не достаточно?
created_at: 2025-03-19T18:19:47Z
author_association: COLLABORATOR
diff_hunk: @@ -83,11 +82,19 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* dCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
+
+        AllDevices.emplace_back(dCtx->DeviceIdx);
 
         if (status == NRdma::RDMA_PROTO_OK) {
             HandleResult(buffer);
         } else {
-            Error = NRdma::ParseError(buffer);
+            auto err = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutError(err)) {
+                ErrDevices.emplace_back(dCtx->DeviceIdx);
+                SendDeviceTimedOut(std::move(dCtx->DeviceUUID));
+            }
+            Error = std::move(err);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: vladstepanyuk
body: попытался вынести часть логики в класс IRdmaDeviceRequestHandler
created_at: 2025-03-20T08:56:26Z
author_association: COLLABORATOR
diff_hunk: @@ -22,6 +22,52 @@ LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER);
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void IRdmaDeviceRequestHandler::SendDeviceTimedOut(TString deviceUUID)
+{
+    auto req = std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+        std::move(deviceUUID));
+    auto event = std::make_unique<IEventHandle>(
+        ParentActorId,
+        TActorId(),
+        req.release(),
+        0,
+        0);
+    ActorSystem->Send(event.release());
+}
+
+bool IRdmaDeviceRequestHandler::NeedToNotifyAboutError(const NProto::TError& err)
+{
+    return err.GetCode() == E_RDMA_UNAVAILABLE || err.GetCode() == E_TIMEOUT;
+}
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: sharpeye
body: Это можно в namespace {} положить
created_at: 2025-03-24T09:55:22Z
author_association: COLLABORATOR
diff_hunk: @@ -144,6 +136,7 @@ class TRdmaRequestContext: public NRdma::IClientHandler
         auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
         auto& counters = *completion->Stats.MutableSysChecksumCounters();
         completion->TotalCycles = RequestInfo->GetTotalCycles();
+        AddDeviceIndicesToCompleteEvent(*completion);
 
         timer.Finish();
         completion->ExecCycles = RequestInfo->GetExecCycles();
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: sharpeye
body: Если создание сообщения вынести в шаблон, будет меньше копипасты:
```cpp
template <typename TCompletionEvent>
std::unique_ptr<TCompletionEvent> CreateCompletionEvent(
    NProto::TError error,
    const TRequestInfo& requestInfo)
{
    auto completion = std::make_unique<TCompletionEvent>(std::move(error));

    completion->DeviceIndices = AllDevices;
    completion->ErrorDeviceIndices = ErrDevices;

    completion->TotalCycles = requestInfo.GetTotalCycles();
    completion->ExecCycles = requestInfo.GetExecCycles();

    return completion;
}
```
```cpp
timer.Finish();

using TCompletionEvent =
    TEvNonreplPartitionPrivate::TEvChecksumBlocksCompleted;

auto completion = CreateCompletionEvent<TCompletionEvent>(
    std::move(Error),
    *RequestInfo);

auto& counters = *completion->Stats.MutableSysChecksumCounters();
counters.SetBlocksCount(RequestBlockCount);
```
```cpp
timer.Finish();

using TCompletionEvent =
    TEvNonreplPartitionPrivate::TEvReadBlocksCompleted;

auto completion = CreateCompletionEvent<TCompletionEvent>(
    std::move(Error),
    *RequestInfo);

completion->NonVoidBlockCount = allZeroes ? 0 : RequestBlockCount;
completion->VoidBlockCount = allZeroes ? RequestBlockCount : 0;

auto& counters = *completion->Stats.MutableUserReadCounters();
counters.SetBlocksCount(RequestBlockCount);
```
...
created_at: 2025-03-24T09:56:53Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +342,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (!inserted) {
+        if (timedOutDeviceCtx.VolumeWasNotified) {
+            return;
+        }
+
+        auto timePassedSinceFirstError =
+            ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: мы же собираемся переименовать Timeouted в TimedOut? 

можно в отдельном ПРе
 
created_at: 2025-03-24T11:36:07Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +342,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (!inserted) {
+        if (timedOutDeviceCtx.VolumeWasNotified) {
+            return;
+        }
+
+        auto timePassedSinceFirstError =
+            ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
+                    deviceUUID));
+            timedOutDeviceCtx.VolumeWasNotified = true;
+        }
+        return;
+    }
+
+    timedOutDeviceCtx.FirstErrorTs = ctx.Now();
+    timedOutDeviceCtx.VolumeWasNotified = false;
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompleted(
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    for (auto idx: opCompleted.DeviceIndices) {
+        Y_ABORT_UNLESS(
+            static_cast<unsigned int>(PartConfig->GetDevices().size()) > idx);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: уже не первый раз вижу size > idx, расскажите плиз какой в этом физический смысл?

idx < size выглядит привычнее
created_at: 2025-03-24T11:37:30Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +20,52 @@ using namespace NActors;
 
 LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER);
 
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool NeedToNotifyAboutDeviceRequestError(const NProto::TError& err)
+{
+    return err.GetCode() == E_RDMA_UNAVAILABLE || err.GetCode() == E_TIMEOUT;
+}
+
+}   // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+void IRdmaDeviceRequestHandler::SendDeviceTimedOut(TString deviceUUID)
+{
+    auto req = std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: sharpeye
body: TCompletionEvent тоже летит в партишен - в обработчике этого эвента нельзя понять, что девайс затаймаутился?
created_at: 2025-03-24T13:43:45Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +342,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (!inserted) {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: sharpeye
body: Удобнее будет инвертировать:
```cpp
if (inserted) {
    timedOutDeviceCtx.FirstErrorTs = ctx.Now();
    timedOutDeviceCtx.VolumeWasNotified = false;
    return;
}
...
```
created_at: 2025-03-24T19:09:05Z
author_association: COLLABORATOR
diff_hunk: @@ -27,14 +28,80 @@ namespace NCloud::NBlockStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceReadRequestContext: public NRdma::TNullContext
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    TString DeviceUUID;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: sharpeye
body: Это где используется?
created_at: 2025-03-24T20:43:16Z
author_association: COLLABORATOR
diff_hunk: @@ -27,14 +28,80 @@ namespace NCloud::NBlockStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceReadRequestContext: public NRdma::TNullContext
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    TString DeviceUUID;
+    ui32 DeviceIdx;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: sharpeye
body: = 0
created_at: 2025-03-24T20:44:18Z
author_association: COLLABORATOR
diff_hunk: @@ -43,18 +45,13 @@ struct TPartialChecksum
 
 ////////////////////////////////////////////////////////////////////////////////
 
-class TRdmaRequestContext: public NRdma::IClientHandler
+class TRdmaRequestContext: public IRdmaDeviceRequestHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: sharpeye
body: TDeviceChecksumRequestHandler
created_at: 2025-03-24T20:54:36Z
author_association: COLLABORATOR
diff_hunk: @@ -102,14 +101,9 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* dCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
 
-        if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(buffer);
-        } else {
-            Error = NRdma::ParseError(buffer);
-        }
-
-        if (--ResponseCount != 0) {
+        if (!ProcessResponse(*dCtx, status, buffer)) {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
user: login: sharpeye
body: HandleResponse, ProcessResponse, HandleResult - можно запутаться. М.б. переименовть HandleResult в ProcessDeviceProto/ProcessResponseProto?
created_at: 2025-03-24T21:04:27Z
author_association: COLLABORATOR
diff_hunk: @@ -126,19 +122,14 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
-        if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dr, buffer);
-        } else {
-            *Response.MutableError() = NRdma::ParseError(buffer);
-        }
-
-        if (--ResponseCount != 0) {
+        if (!ProcessResponse(*dr, status, buffer)) {
             return;
         }
 
         // Got all device responses. Do processing.
 
-        ProcessError(*ActorSystem, *PartConfig, *Response.MutableError());
+        ProcessError(*ActorSystem, *PartConfig, Error);
+        *Response.MutableError() = std::move(Error);
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: sharpeye
body: `Error` ниже используется
created_at: 2025-03-24T21:18:12Z
author_association: COLLABORATOR
diff_hunk: @@ -52,20 +47,22 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
             NActors::TActorId parentActorId,
             ui64 requestId,
             bool checkVoidBlocks)
-        : ActorSystem(actorSystem)
+        : IRdmaDeviceRequestHandler(
+              requestCount,
+              actorSystem,
+              parentActorId,
+              requestBlockCount)
         , PartConfig(std::move(partConfig))
         , RequestInfo(std::move(requestInfo))
-        , RequestBlockCount(requestBlockCount)
-        , ParentActorId(parentActorId)
         , RequestId(requestId)
         , CheckVoidBlocks(checkVoidBlocks)
-        , ResponseCount(requestCount)
         , SgList(std::move(sglist))
     {
     }
 
-    void HandleResult(const TDeviceReadRequestContext& dr, TStringBuf buffer)
+    void HandleResult(const TDeviceRequestContext& dCtx, TStringBuf buffer) override
     {
+        const auto& dr = static_cast<const TDeviceReadRequestContext&>(dCtx);
         if (auto guard = SgList.Acquire()) {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: sharpeye
body: не должно ли тут быть E_CANCELLED, в [случае](https://github.com/ydb-platform/nbs/blob/main/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_readblocks_local.cpp#L162) `!guard` ?
created_at: 2025-03-24T21:21:20Z
author_association: COLLABORATOR
diff_hunk: @@ -294,6 +342,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (!inserted) {
+        if (timedOutDeviceCtx.VolumeWasNotified) {
+            return;
+        }
+
+        auto timePassedSinceFirstError =
+            ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+        if (timePassedSinceFirstError >=
+            Config->GetLaggingDeviceTimeoutThreshold())
+        {
+            NCloud::Send(
+                ctx,
+                PartConfig->GetParentActorId(),
+                std::make_unique<TEvVolumePrivate::TEvDeviceTimeoutedRequest>(
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: https://github.com/ydb-platform/nbs/pull/3250
created_at: 2025-03-25T09:58:56Z
author_association: COLLABORATOR
diff_hunk: @@ -67,6 +129,11 @@ class TNonreplicatedPartitionRdmaActor final
 
     TRequestInfoPtr Poisoner;
 
+    struct TTimedOutDeviceCtx {
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: sharpeye
body: { не там

Можно назвать TDeviceStat, как [тут](https://github.com/ydb-platform/nbs/blob/main/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor.h#L65), и работать аналогичным образом:
```cpp
TVector<TDeviceStat> DeviceStats;
```
created_at: 2025-03-27T09:47:02Z
author_association: COLLABORATOR
diff_hunk: @@ -25,18 +25,15 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
+class TRdmaRequestReadBlocksContext: public IRdmaDeviceRequestHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: sharpeye
body: TRdmaRequestReadBlocksHandler, остальные тоже надо переименовать с *Context на *Handler
created_at: 2025-03-31T14:49:26Z
author_association: COLLABORATOR
diff_hunk: @@ -27,14 +28,74 @@ namespace NCloud::NBlockStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDeviceReadRequestContext: public NRdma::TNullContext
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    ui32 DeviceIdx = 0;
+};
+
+struct TDeviceReadRequestContext: public TDeviceRequestContext
 {
     ui64 StartIndexOffset = 0;
     ui64 BlockCount = 0;
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 
+class IRdmaDeviceRequestHandler: public NRdma::IClientHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h
user: login: sharpeye
body: Можно попробовать CRTP - шаблон, который делает всю общую обработку, различия реализуются вызовом кода наследника:
```cpp
template <typename TDerived>
class TRdmaDeviceRequestHandler: public NRdma::IClientHandler
{
private:
    NActors::TActorSystem* ActorSystem;
    const TNonreplicatedPartitionConfigPtr PartConfig;
    const TRequestInfoPtr RequestInfo;
    const ui64 RequestId;
    const NActors::TActorId ParentActorId;
    const ui32 RequestBlockCount;

    ui32 ResponseCount;
    TStackVec<ui32, 2> AllDevices;
    TStackVec<ui32, 2> ErrDevices;

    TAdaptiveLock Lock;
    NProto::TError Error;

public:
    ...
    template <typename TCompletionEvent>
    std::unique_ptr<TCompletionEvent> CreateCompletionEvent() const
    {
        ...
    }

    bool HandleSubResponse(
        NRdma::TClientRequestPtr req,
        ui32 status,
        size_t responseBytes)
    {
        ...
        auto error = static_cast<TDerived*>(this)->ProcessSubResponse(*dr, buffer);
        ...
    }

    void HandleResponse(
        NRdma::TClientRequestPtr req,
        ui32 status,
        size_t responseBytes) override
    {
        TRequestScope timer(*RequestInfo);

        auto guard = Guard(Lock);

        if (HandleSubResponse(std::move(req), status, responseBytes)) {
            return;
        }

        auto completion = static_cast<const TDerived*>(this)->CreateCompletionEvent();
        auto completionEvent = std::make_unique<NActors::IEventHandle>(
            ...
            completion.release(),
            ...
            );

        auto response = static_cast<TDerived*>(this)->CreateResponse(std::move(Error));
        auto event = std::make_unique<NActors::IEventHandle>(
            ...
            response.release()
            ...
            );

        timer.Finish();

        ActorSystem->Send(event.release());
        ActorSystem->Send(completionEvent.release());
    }
    ...
// всё, что нужно наследнику - выставляем функциями:
   TRequestInfo& GetRequestInfo()
    {
        return *RequestInfo;
    }

    const TNonreplicatedPartitionConfig& GetPartConfig() const
    {
        return *PartConfig;
    }
    ...
};
```
Наследники определяют только свои куски - парсинг протобуфов и создание эвентов:
```cpp
class TRdmaWriteBlocksResponseHandler
    : public TRdmaDeviceRequestHandler<TRdmaWriteBlocksResponseHandler>
{
    using TBase = TRdmaDeviceRequestHandler<TRdmaWriteBlocksResponseHandler>;
private:
    const bool ReplyLocal;

public:
    TRdmaWriteBlocksResponseHandler(...): TRdmaDeviceRequestHandler(...)
    {}

    std::unique_ptr<IEventBase> CreateResponse(const NProto::TError& error) const
    {
        if (ReplyLocal) {
            return std::make_unique<TEvService::TEvWriteBlocksLocalResponse>(error);
        }

        return std::make_unique<TEvService::TEvWriteBlocksResponse>(error);
    }

    auto CreateCompletionEvent() const
    {
        auto completion = TBase::CreateCompletionEvent<TEvNonreplPartitionPrivate::TEvWriteBlocksCompleted>();
        ...
    }

    NProto::TError ProcessSubResponse(
        const TDeviceRequestContext& context,
        TStringBuf buffer) const
    {
        ...
        const auto& concreteProto = static_cast<NProto::TWriteDeviceBlocksResponse&>(*result.Proto);
        return concreteProto.GetError();
    }
};
```

created_at: 2025-04-02T13:03:41Z
author_association: COLLABORATOR
diff_hunk: @@ -43,120 +45,84 @@ struct TPartialChecksum
 
 ////////////////////////////////////////////////////////////////////////////////
 
-class TRdmaRequestContext: public NRdma::IClientHandler
+class TDeviceChecksumRequestHandler
+    : public TRdmaDeviceRequestHandler<TDeviceChecksumRequestHandler>
 {
+    using TBase = TRdmaDeviceRequestHandler<TDeviceChecksumRequestHandler>;
+
 private:
-    TActorSystem* ActorSystem;
-    TNonreplicatedPartitionConfigPtr PartConfig;
-    TRequestInfoPtr RequestInfo;
-    TAdaptiveLock Lock;
-    size_t ResponseCount;
     TMap<ui64, TPartialChecksum> Checksums;
-    NProto::TError Error;
-    const ui32 RequestBlockCount;
-    NActors::TActorId ParentActorId;
-    ui64 RequestId;
 
 public:
-    TRdmaRequestContext(
+    TDeviceChecksumRequestHandler(
             TActorSystem* actorSystem,
             TNonreplicatedPartitionConfigPtr partConfig,
             TRequestInfoPtr requestInfo,
             size_t requestCount,
             ui32 requestBlockCount,
             NActors::TActorId parentActorId,
             ui64 requestId)
-        : ActorSystem(actorSystem)
-        , PartConfig(std::move(partConfig))
-        , RequestInfo(std::move(requestInfo))
-        , ResponseCount(requestCount)
-        , RequestBlockCount(requestBlockCount)
-        , ParentActorId(parentActorId)
-        , RequestId(requestId)
+        : TRdmaDeviceRequestHandler(
+              actorSystem,
+              std::move(partConfig),
+              std::move(requestInfo),
+              requestId,
+              parentActorId,
+              requestBlockCount,
+              requestCount)
+
     {}
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: sharpeye
body: ```
using TRdmaDeviceRequestHandler::TRdmaDeviceRequestHandler;
```

created_at: 2025-04-04T09:26:29Z
author_association: COLLABORATOR
diff_hunk: @@ -112,70 +109,39 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
         if (isAllZeroes) {
             VoidBlockCount += dr.BlockCount;
         }
+
+        return {};
     }
 
-    void HandleResponse(
-        NRdma::TClientRequestPtr req,
-        ui32 status,
-        size_t responseBytes) override
+    std::unique_ptr<TEvNonreplPartitionPrivate::TEvReadBlocksCompleted>
+    CreateCompletionEvent() const
     {
-        TRequestScope timer(*RequestInfo);
-
-        auto guard = Guard(Lock);
-
-        auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
-        auto buffer = req->ResponseBuffer.Head(responseBytes);
-
-        if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dr, buffer);
-        } else {
-            *Response.MutableError() = NRdma::ParseError(buffer);
-        }
+        const ui32 blockCount = Response.GetBlocks().BuffersSize();
+        const bool allZeroes = VoidBlockCount == blockCount;
 
-        if (--ResponseCount != 0) {
-            return;
-        }
+        auto completion = TBase::CreateCompletionEvent<
+            TEvNonreplPartitionPrivate::TEvReadBlocksCompleted>();
 
-        // Got all device responses. Do processing.
+        completion->NonVoidBlockCount = allZeroes ? 0 : blockCount;
+        completion->VoidBlockCount = allZeroes ? blockCount : 0;
+        auto& counters = *completion->Stats.MutableUserReadCounters();
+        counters.SetBlocksCount(blockCount);
 
-        ProcessError(*ActorSystem, *PartConfig, *Response.MutableError());
-        auto error = Response.GetError();
+        return completion;
+    }
 
+    std::unique_ptr<TEvService::TEvReadBlocksResponse> CreateResponse(
+        NProto::TError err)
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: sharpeye
body: error
created_at: 2025-04-04T09:28:24Z
author_association: COLLABORATOR
diff_hunk: @@ -52,124 +45,97 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
             NActors::TActorId parentActorId,
             ui64 requestId,
             bool checkVoidBlocks)
-        : ActorSystem(actorSystem)
-        , PartConfig(std::move(partConfig))
-        , RequestInfo(std::move(requestInfo))
-        , RequestBlockCount(requestBlockCount)
-        , ParentActorId(parentActorId)
-        , RequestId(requestId)
+        : TRdmaDeviceRequestHandler(
+              actorSystem,
+              std::move(partConfig),
+              std::move(requestInfo),
+              requestId,
+              parentActorId,
+              requestBlockCount,
+              requestCount)
         , CheckVoidBlocks(checkVoidBlocks)
-        , ResponseCount(requestCount)
         , SgList(std::move(sglist))
-    {
-    }
+    {}
 
-    void HandleResult(const TDeviceReadRequestContext& dr, TStringBuf buffer)
+    NProto::TError ProcessSubResponse(
+        const TDeviceRequestContext& dCtx,
+        TStringBuf buffer)
     {
-        if (auto guard = SgList.Acquire()) {
-            auto* serializer = TBlockStoreProtocol::Serializer();
-            auto [result, err] = serializer->Parse(buffer);
-
-            if (HasError(err)) {
-                Error = std::move(err);
-                return;
-            }
-
-            const auto& concreteProto =
-                static_cast<NProto::TReadDeviceBlocksResponse&>(*result.Proto);
-            if (HasError(concreteProto.GetError())) {
-                Error = concreteProto.GetError();
-                return;
-            }
+        const auto& dr = static_cast<const TDeviceReadRequestContext&>(dCtx);
+        auto guard = SgList.Acquire();
+        if (!guard) {
+            return MakeError(E_CANCELLED, "can't acquire sglist");
+        }
+        auto* serializer = TBlockStoreProtocol::Serializer();
+        auto [result, err] = serializer->Parse(buffer);
 
-            TSgList data = guard.Get();
+        if (HasError(err)) {
+            return err;
+        }
 
-            ui64 offset = 0;
-            ui64 b = 0;
-            bool isAllZeroes = CheckVoidBlocks;
-            while (offset < result.Data.size()) {
-                ui64 targetBlock = dr.StartIndexOffset + b;
-                Y_ABORT_UNLESS(targetBlock < data.size());
-                ui64 bytes =
-                    Min(result.Data.size() - offset, data[targetBlock].Size());
-                Y_ABORT_UNLESS(bytes);
+        const auto& concreteProto =
+            static_cast<NProto::TReadDeviceBlocksResponse&>(*result.Proto);
+        if (HasError(concreteProto.GetError())) {
+            return concreteProto.GetError();
+        }
 
-                char* dst = const_cast<char*>(data[targetBlock].Data());
-                const char* src = result.Data.data() + offset;
+        TSgList data = guard.Get();
 
-                if (isAllZeroes) {
-                    isAllZeroes = IsAllZeroes(src, bytes);
-                }
-                memcpy(dst, src, bytes);
+        ui64 offset = 0;
+        ui64 b = 0;
+        bool isAllZeroes = CheckVoidBlocks;
+        while (offset < result.Data.size()) {
+            ui64 targetBlock = dr.StartIndexOffset + b;
+            Y_ABORT_UNLESS(targetBlock < data.size());
+            ui64 bytes =
+                Min(result.Data.size() - offset, data[targetBlock].Size());
+            Y_ABORT_UNLESS(bytes);
 
-                offset += bytes;
-                ++b;
-            }
+            char* dst = const_cast<char*>(data[targetBlock].Data());
+            const char* src = result.Data.data() + offset;
 
             if (isAllZeroes) {
-                VoidBlockCount += dr.BlockCount;
+                isAllZeroes = IsAllZeroes(src, bytes);
             }
-        }
-    }
-
-    void HandleResponse(
-        NRdma::TClientRequestPtr req,
-        ui32 status,
-        size_t responseBytes) override
-    {
-        TRequestScope timer(*RequestInfo);
-
-        auto guard = Guard(Lock);
+            memcpy(dst, src, bytes);
 
-        auto* dr = static_cast<TDeviceReadRequestContext*>(req->Context.get());
-        auto buffer = req->ResponseBuffer.Head(responseBytes);
-
-        if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dr, buffer);
-        } else {
-            Error = NRdma::ParseError(buffer);
+            offset += bytes;
+            ++b;
         }
 
-        if (--ResponseCount != 0) {
-            return;
+        if (isAllZeroes) {
+            VoidBlockCount += dr.BlockCount;
         }
 
-        // Got all device responses. Do processing.
-
-        ProcessError(*ActorSystem, *PartConfig, Error);
+        return {};
+    }
 
+    std::unique_ptr<TEvNonreplPartitionPrivate::TEvReadBlocksCompleted>
+    CreateCompletionEvent() const
+    {
+        const auto RequestBlockCount = GetRequestBlockCount();
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: sharpeye
body: ```
requestBlockCount
```
created_at: 2025-04-04T09:29:37Z
author_association: COLLABORATOR
diff_hunk: @@ -25,105 +25,66 @@ using TResponse = TEvService::TEvZeroBlocksResponse;
 
 ////////////////////////////////////////////////////////////////////////////////
 
-class TRdmaRequestContext: public NRdma::IClientHandler
+class TRdmaRequestZeroBlocksHandler
+    : public TRdmaDeviceRequestHandler<TRdmaRequestZeroBlocksHandler>
 {
-private:
-    TActorSystem* ActorSystem;
-    TNonreplicatedPartitionConfigPtr PartConfig;
-    TRequestInfoPtr RequestInfo;
-    TAdaptiveLock Lock;
-    size_t ResponseCount;
-    NProto::TError Error;
-    const ui32 RequestBlockCount;
-    NActors::TActorId ParentActorId;
-    ui64 RequestId;
+    using TBase = TRdmaDeviceRequestHandler<TRdmaRequestZeroBlocksHandler>;
 
 public:
-    TRdmaRequestContext(
+    TRdmaRequestZeroBlocksHandler(
             TActorSystem* actorSystem,
             TNonreplicatedPartitionConfigPtr partConfig,
             TRequestInfoPtr requestInfo,
             size_t requestCount,
             ui32 requestBlockCount,
             NActors::TActorId parentActorId,
             ui64 requestId)
-        : ActorSystem(actorSystem)
-        , PartConfig(std::move(partConfig))
-        , RequestInfo(std::move(requestInfo))
-        , ResponseCount(requestCount)
-        , RequestBlockCount(requestBlockCount)
-        , ParentActorId(parentActorId)
-        , RequestId(requestId)
+        : TRdmaDeviceRequestHandler(
+              actorSystem,
+              std::move(partConfig),
+              std::move(requestInfo),
+              requestId,
+              parentActorId,
+              requestBlockCount,
+              requestCount)
     {}
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: sharpeye
body: ```
using TRdmaDeviceRequestHandler::TRdmaDeviceRequestHandler;
```
created_at: 2025-04-04T09:30:25Z
author_association: COLLABORATOR
diff_hunk: @@ -25,105 +25,66 @@ using TResponse = TEvService::TEvZeroBlocksResponse;
 
 ////////////////////////////////////////////////////////////////////////////////
 
-class TRdmaRequestContext: public NRdma::IClientHandler
+class TRdmaRequestZeroBlocksHandler
+    : public TRdmaDeviceRequestHandler<TRdmaRequestZeroBlocksHandler>
 {
-private:
-    TActorSystem* ActorSystem;
-    TNonreplicatedPartitionConfigPtr PartConfig;
-    TRequestInfoPtr RequestInfo;
-    TAdaptiveLock Lock;
-    size_t ResponseCount;
-    NProto::TError Error;
-    const ui32 RequestBlockCount;
-    NActors::TActorId ParentActorId;
-    ui64 RequestId;
+    using TBase = TRdmaDeviceRequestHandler<TRdmaRequestZeroBlocksHandler>;
 
 public:
-    TRdmaRequestContext(
+    TRdmaRequestZeroBlocksHandler(
             TActorSystem* actorSystem,
             TNonreplicatedPartitionConfigPtr partConfig,
             TRequestInfoPtr requestInfo,
             size_t requestCount,
             ui32 requestBlockCount,
             NActors::TActorId parentActorId,
             ui64 requestId)
-        : ActorSystem(actorSystem)
-        , PartConfig(std::move(partConfig))
-        , RequestInfo(std::move(requestInfo))
-        , ResponseCount(requestCount)
-        , RequestBlockCount(requestBlockCount)
-        , ParentActorId(parentActorId)
-        , RequestId(requestId)
+        : TRdmaDeviceRequestHandler(
+              actorSystem,
+              std::move(partConfig),
+              std::move(requestInfo),
+              requestId,
+              parentActorId,
+              requestBlockCount,
+              requestCount)
     {}
 
-    void HandleResult(TStringBuf buffer)
+    NProto::TError ProcessSubResponse(
+        const TDeviceRequestContext& dCtx,
+        TStringBuf buffer) const
     {
+        Y_UNUSED(dCtx);
         auto* serializer = TBlockStoreProtocol::Serializer();
         auto [result, err] = serializer->Parse(buffer);
 
         if (HasError(err)) {
-            Error = std::move(err);
-            return;
+            return err;
         }
 
         const auto& concreteProto =
             static_cast<NProto::TZeroDeviceBlocksResponse&>(*result.Proto);
         if (HasError(concreteProto.GetError())) {
-            Error = concreteProto.GetError();
+            return concreteProto.GetError();
         }
+
+        return {};
     }
 
-    void HandleResponse(
-        NRdma::TClientRequestPtr req,
-        ui32 status,
-        size_t responseBytes) override
+    std::unique_ptr<TEvNonreplPartitionPrivate::TEvZeroBlocksCompleted>
+    CreateCompletionEvent() const
     {
-        TRequestScope timer(*RequestInfo);
-
-        auto guard = Guard(Lock);
-
-        auto buffer = req->ResponseBuffer.Head(responseBytes);
-
-        if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(buffer);
-        } else {
-            Error = NRdma::ParseError(buffer);
-        }
-
-        if (--ResponseCount != 0) {
-            return;
-        }
-
-        // Got all device responses. Do processing.
-
-        ProcessError(*ActorSystem, *PartConfig, Error);
-
-        auto response = std::make_unique<TResponse>(Error);
-        auto event = std::make_unique<IEventHandle>(
-            RequestInfo->Sender,
-            TActorId(),
-            response.release(),
-            0,
-            RequestInfo->Cookie);
-        ActorSystem->Send(event.release());
-
-        using TCompletionEvent =
-            TEvNonreplPartitionPrivate::TEvZeroBlocksCompleted;
-        auto completion = std::make_unique<TCompletionEvent>(std::move(Error));
+        auto completion = TBase::CreateCompletionEvent<
+            TEvNonreplPartitionPrivate::TEvZeroBlocksCompleted>();
         auto& counters = *completion->Stats.MutableUserWriteCounters();
-        completion->TotalCycles = RequestInfo->GetTotalCycles();
-
-        timer.Finish();
-        completion->ExecCycles = RequestInfo->GetExecCycles();
-
-        counters.SetBlocksCount(RequestBlockCount);
-        auto completionEvent = std::make_unique<IEventHandle>(
-            ParentActorId,
-            TActorId(),
-            completion.release(),
-            0,
-            RequestId);
-        ActorSystem->Send(completionEvent.release());
+        counters.SetBlocksCount(GetRequestBlockCount());
+        return completion;
+    }
+
+    std::unique_ptr<TEvService::TEvZeroBlocksResponse> CreateResponse(
+        NProto::TError err) const
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: sharpeye
body: error
created_at: 2025-04-04T09:30:50Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,151 @@
+#pragma once
+
+#include <cloud/blockstore/libs/rdma/iface/client.h>
+#include <cloud/blockstore/libs/rdma/iface/protobuf.h>
+#include <cloud/blockstore/libs/rdma/iface/protocol.h>
+#include <cloud/blockstore/libs/storage/core/request_info.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_common.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/public.h>
+
+#include <contrib/ydb/library/actors/core/actorsystem.h>
+
+namespace NCloud::NBlockStore::NStorage {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    ui32 DeviceIdx = 0;
+};
+
+template <typename TDerived>
+class TRdmaDeviceRequestHandler: public NRdma::IClientHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h
user: login: sharpeye
body: Давай на два PR распилим - рефакторинг и уведомления. 
created_at: 2025-04-04T10:50:18Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,151 @@
+#pragma once
+
+#include <cloud/blockstore/libs/rdma/iface/client.h>
+#include <cloud/blockstore/libs/rdma/iface/protobuf.h>
+#include <cloud/blockstore/libs/rdma/iface/protocol.h>
+#include <cloud/blockstore/libs/storage/core/request_info.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_common.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/public.h>
+
+#include <contrib/ydb/library/actors/core/actorsystem.h>
+
+namespace NCloud::NBlockStore::NStorage {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    ui32 DeviceIdx = 0;
+};
+
+template <typename TDerived>
+class TRdmaDeviceRequestHandler: public NRdma::IClientHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h
user: login: vladstepanyuk
body: мне кажется уже поздно этот пр пилить) можно эту версию залить а рефакторинг которого еще хочется отдельным пром
created_at: 2025-04-09T14:51:29Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,151 @@
+#pragma once
+
+#include <cloud/blockstore/libs/rdma/iface/client.h>
+#include <cloud/blockstore/libs/rdma/iface/protobuf.h>
+#include <cloud/blockstore/libs/rdma/iface/protocol.h>
+#include <cloud/blockstore/libs/storage/core/request_info.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_common.h>
+#include <cloud/blockstore/libs/storage/partition_nonrepl/public.h>
+
+#include <contrib/ydb/library/actors/core/actorsystem.h>
+
+namespace NCloud::NBlockStore::NStorage {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TDeviceRequestContext: public NRdma::TNullContext
+{
+    ui32 DeviceIdx = 0;
+};
+
+template <typename TDerived>
+class TRdmaDeviceRequestHandler: public NRdma::IClientHandler
path: cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h
user: login: sharpeye
body: Не поздно.
В этом PR сделан рефакторинг (вынос кода в rdma_device_request_handler.h) и добавлена новая фича - уведомления про таймауты. Надо сделать из одного PR два - отдельно реыфакторинг, отдельно фича.

created_at: 2025-04-14T10:40:34Z
author_association: COLLABORATOR
diff_hunk: @@ -300,6 +308,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (inserted) {
+        timedOutDeviceCtx.FirstErrorTs = ctx.Now();
+        timedOutDeviceCtx.VolumeWasNotified = false;
+        return;
+    }
+
+    if (timedOutDeviceCtx.VolumeWasNotified) {
+        return;
+    }
+
+    auto timePassedSinceFirstError = ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+    if (timePassedSinceFirstError >= Config->GetLaggingDeviceTimeoutThreshold())
+    {
+        NCloud::Send(
+            ctx,
+            PartConfig->GetParentActorId(),
+            std::make_unique<TEvVolumePrivate::TEvDeviceTimedOutRequest>(
+                deviceUUID));
+        timedOutDeviceCtx.VolumeWasNotified = true;
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompleted(
+    const NActors::TActorContext& ctx,
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    const auto& devices = PartConfig->GetDevices();
+    for (auto idx: opCompleted.DeviceIndices) {
+        Y_ABORT_UNLESS(idx < static_cast<unsigned int>(devices.size()));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: SvartMetal
body: static_cast тут точно нужен?

и не unsigned int, а ui32
created_at: 2025-04-22T19:03:40Z
author_association: COLLABORATOR
diff_hunk: @@ -41,6 +41,9 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
 
     ui32 VoidBlockCount = 0;
 
+    TStackVec<ui32, 2> AllDevices;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: SvartMetal
body: DeviceIndices

как тут https://github.com/ydb-platform/nbs/blob/4d8bdfb6f7b5d8b00136214ad1309244ce4e7ffd/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h#L221

и аналогичный комментарий
created_at: 2025-04-22T19:31:03Z
author_association: COLLABORATOR
diff_hunk: @@ -219,6 +219,7 @@ struct TEvNonreplPartitionPrivate
 
         // Indexes of devices that participated in the request.
         TStackVec<ui32, 2> DeviceIndices;
+        TStackVec<ui32, 2> ErrorDeviceIndices;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h
user: login: SvartMetal
body: комментарий
created_at: 2025-04-22T19:32:03Z
author_association: COLLABORATOR
diff_hunk: @@ -41,6 +41,9 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
 
     ui32 VoidBlockCount = 0;
 
+    TStackVec<ui32, 2> AllDevices;
+    TStackVec<ui32, 2> ErrDevices;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: SvartMetal
body: ErrorDeviceIndices
created_at: 2025-04-22T19:32:49Z
author_association: COLLABORATOR
diff_hunk: @@ -102,11 +105,17 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
         auto guard = Guard(Lock);
 
         auto buffer = req->ResponseBuffer.Head(responseBytes);
+        auto* dCtx = static_cast<TDeviceRequestContext*>(req->Context.get());
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
user: login: SvartMetal
body: где-то эта штука называется dc, где-то dr, тут dCtx

давай везде сделаем одинаково, например reqCtx
created_at: 2025-04-22T19:33:49Z
author_association: COLLABORATOR
diff_hunk: @@ -571,6 +578,147 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest)
         UNIT_ASSERT_VALUES_EQUAL(1, notificationCount);
     }
 
+    Y_UNIT_TEST(ShouldHandleRequestSendFailure)
+    {
+        TTestBasicRuntime runtime;
+
+        TTestEnv env(
+            runtime,
+            NProto::VOLUME_IO_OK,
+            TTestEnv::DefaultDevices(runtime.GetNodeId(0)),
+            false,
+            true);
+        TPartitionClient client(runtime, env.ActorId);
+
+        TActorId notifiedActor;
+        ui32 deviceTimedOut = 0;
+        THashSet<TString> devices;
+
+        runtime.SetObserverFunc(
+            [&](TAutoPtr<IEventHandle>& event)
+            {
+                switch (event->GetTypeRewrite()) {
+                    case TEvVolumePrivate::EvDeviceTimedOutRequest: {
+                        if (event->Recipient != env.VolumeActorId) {
+                            break;
+                        }
+                        notifiedActor = event->Recipient;
+                        auto* ev = static_cast<
+                            TEvVolumePrivate::TEvDeviceTimedOutRequest*>(
+                            event->GetBase());
+                        devices.emplace(ev->DeviceUUID);
+                        ++deviceTimedOut;
+                    }
+                }
+
+                return TTestActorRuntime::DefaultObserverFunc(event);
+            });
+
+        env.Rdma().InjectErrors(
+            {},
+            MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable"),
+            {});
+        env.Rdma().InitAllEndpoints();
+        const auto error = MakeError(E_RDMA_UNAVAILABLE, "rdma unavailable");
+
+        const auto blockRange1 = TBlockRange64::WithLength(1024, 3072);
+
+        WRITE_BLOCKS_E(error);
+        READ_BLOCKS_E(error, 0);
+        UNIT_ASSERT_VALUES_EQUAL(0, devices.size());
+        runtime.DispatchEvents({}, 10ms);
+        runtime.AdvanceCurrentTime(
+            env.Config->GetLaggingDeviceTimeoutThreshold() + 1ms);
+        ZERO_BLOCKS_E(error);
+        runtime.DispatchEvents({}, 10ms);
+
+        THashSet<TString> expected = {"vasya", "petya"};
+        UNIT_ASSERT_VALUES_EQUAL(expected.size(), devices.size());
+        for (const auto& d: devices) {
+            UNIT_ASSERT(expected.contains(d));
+        }
+
+        UNIT_ASSERT_VALUES_EQUAL(2, deviceTimedOut);
+    }
+
+    Y_UNIT_TEST(ShouldResetDeviceTimeoutInfoOnSucceededRequest)
+    {
+        TTestBasicRuntime runtime;
+
+        TTestEnv env(
+            runtime,
+            NProto::VOLUME_IO_OK,
+            TTestEnv::DefaultDevices(runtime.GetNodeId(0)),
+            false,
+            true);
+        TPartitionClient client(runtime, env.ActorId);
+
+        TActorId notifiedActor;
+        ui32 deviceTimedOut = 0;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp
user: login: SvartMetal
body: devicesTimedOut

или deviceTimedOutCount
created_at: 2025-04-22T19:37:07Z
author_association: COLLABORATOR
diff_hunk: @@ -300,6 +308,58 @@ NProto::TError TNonreplicatedPartitionRdmaActor::SendReadRequests(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void TNonreplicatedPartitionRdmaActor::NotifyDeviceTimedOutIfNeeded(
+    const NActors::TActorContext& ctx,
+    const TString& deviceUUID)
+{
+    if (!PartConfig->GetLaggingDevicesAllowed()) {
+        return;
+    }
+    auto [it, inserted] = TimedOutDeviceCtxByDeviceUUID.try_emplace(
+        deviceUUID,
+        TTimedOutDeviceCtx{});
+    auto& timedOutDeviceCtx = it->second;
+
+    if (inserted) {
+        timedOutDeviceCtx.FirstErrorTs = ctx.Now();
+        timedOutDeviceCtx.VolumeWasNotified = false;
+        return;
+    }
+
+    if (timedOutDeviceCtx.VolumeWasNotified) {
+        return;
+    }
+
+    auto timePassedSinceFirstError = ctx.Now() - timedOutDeviceCtx.FirstErrorTs;
+    if (timePassedSinceFirstError >= Config->GetLaggingDeviceTimeoutThreshold())
+    {
+        NCloud::Send(
+            ctx,
+            PartConfig->GetParentActorId(),
+            std::make_unique<TEvVolumePrivate::TEvDeviceTimedOutRequest>(
+                deviceUUID));
+        timedOutDeviceCtx.VolumeWasNotified = true;
+    }
+}
+
+void TNonreplicatedPartitionRdmaActor::ProcessOperationCompleted(
+    const NActors::TActorContext& ctx,
+    const TEvNonreplPartitionPrivate::TOperationCompleted& opCompleted)
+{
+    const auto& devices = PartConfig->GetDevices();
+    for (auto idx: opCompleted.DeviceIndices) {
+        Y_ABORT_UNLESS(idx < static_cast<unsigned int>(devices.size()));
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp
user: login: vladstepanyuk
body: без статик каста код не компилируется

```
error: comparison of integers of different signs: 'unsigned int' and 'int' [-Werror,-Wsign-compare]
        Y_ABORT_UNLESS(idx < devices.size());
```
created_at: 2025-04-23T06:09:35Z
author_association: COLLABORATOR
diff_hunk: @@ -38,9 +38,16 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
     TAdaptiveLock Lock;
     size_t ResponseCount;
     NProto::TReadBlocksResponse Response;
+    NProto::TError Error;
 
     ui32 VoidBlockCount = 0;
 
+    // Indexes of devices that participated in the request.
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: SvartMetal
body: Indices (в названии ведь так)
created_at: 2025-04-23T14:58:02Z
author_association: COLLABORATOR
diff_hunk: @@ -38,9 +38,16 @@ class TRdmaRequestReadBlocksContext: public NRdma::IClientHandler
     TAdaptiveLock Lock;
     size_t ResponseCount;
     NProto::TReadBlocksResponse Response;
+    NProto::TError Error;
 
     ui32 VoidBlockCount = 0;
 
+    // Indexes of devices that participated in the request.
+    TStackVec<ui32, 2> DeviceIndices;
+
+    // Indexes of devices where requests have resulted in errors.
+    TStackVec<ui32, 2> ErrorDeviceIndices;
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks.cpp
user: login: SvartMetal
body: Indices
created_at: 2025-04-23T14:58:09Z
author_association: COLLABORATOR
diff_hunk: @@ -41,6 +41,12 @@ class TRdmaRequestReadBlocksLocalContext: public NRdma::IClientHandler
 
     ui32 VoidBlockCount = 0;
 
+    // Indexes of devices that participated in the request.
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_readblocks_local.cpp
user: login: SvartMetal
body: тоже Indices

(и на 47 строке)
created_at: 2025-04-23T14:58:34Z
author_association: COLLABORATOR
diff_hunk: @@ -45,6 +45,12 @@ class TRdmaWriteBlocksResponseHandler: public NRdma::IClientHandler
     const NActors::TActorId ParentActorId;
     const ui64 RequestId;
 
+    // Indexes of devices that participated in the request.
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp
user: login: SvartMetal
body: и тут
created_at: 2025-04-23T14:58:49Z
author_association: COLLABORATOR
diff_hunk: @@ -38,6 +38,12 @@ class TRdmaRequestContext: public NRdma::IClientHandler
     NActors::TActorId ParentActorId;
     ui64 RequestId;
 
+    // Indexes of devices that participated in the request.
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp
user: login: SvartMetal
body: и тут
created_at: 2025-04-23T14:58:59Z
author_association: COLLABORATOR
diff_hunk: @@ -106,14 +114,19 @@ class TRdmaRequestContext: public NRdma::IClientHandler
 
         auto guard = Guard(Lock);
 
-        auto* dc =
+        auto* reqCtx =
             static_cast<TDeviceChecksumRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dc, buffer);
+            HandleResult(*reqCtx, buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: sharpeye
body: Если вместо DeviceIndices и ErrorDeviceIndices сделать:
```cpp
struct TDeviceResponseResult
{
    ui32 Index;
    NProto::TError Error;
};
TStackVec<TDeviceResponseResult, 2> DeviceResults;
```
то тут можно просто сохранять ошибку для каждого девайса:
```cpp
auto error = status == NRdma::RDMA_PROTO_OK
    ? HandleResult(*reqCtx, buffer)
    : NRdma::ParseError(buffer);
if (HasError(error)) {
    Error = error;
}

DeviceResults.emplace_back(reqCtx->DeviceIdx, std::move(error));

```
а обратывать уже в ProcessOperationCompleted - что логично, т.к. именно там вызывается NotifyDeviceTimedOutIfNeeded:
```cpp
for (const auto& [deviceIndex, error]: ev.DeviceResults) {
    const auto& deviceUUID = devices[deviceIndex].GetDeviceUUID();
    if (NeedToNotifyAboutDeviceRequestError(error)) {
        NotifyDeviceTimedOutIfNeeded(ctx, deviceUUID);
        continue;
    }
    TimedOutDeviceCtxByDeviceUUID.erase(deviceUUID);
}
```
created_at: 2025-04-23T19:00:15Z
author_association: COLLABORATOR
diff_hunk: @@ -106,14 +114,19 @@ class TRdmaRequestContext: public NRdma::IClientHandler
 
         auto guard = Guard(Lock);
 
-        auto* dc =
+        auto* reqCtx =
             static_cast<TDeviceChecksumRequestContext*>(req->Context.get());
         auto buffer = req->ResponseBuffer.Head(responseBytes);
 
+        DeviceIndices.emplace_back(reqCtx->DeviceIdx);
+
         if (status == NRdma::RDMA_PROTO_OK) {
-            HandleResult(*dc, buffer);
+            HandleResult(*reqCtx, buffer);
         } else {
             Error = NRdma::ParseError(buffer);
+            if (NeedToNotifyAboutDeviceRequestError(Error)) {
+                ErrorDeviceIndices.emplace_back(reqCtx->DeviceIdx);
+            }
         }
path: cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp
user: login: vladstepanyuk
body: поправлю в отдельном пре с рефакторингом, т.к. эти изменения не только мой код заденут
created_at: 2025-04-24T04:38:18Z
author_association: COLLABORATOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-14T09:58:25Z
body: Hi! Thank you for contributing!
The tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-14T13:07:34Z
body: <!-- status pr=3187, run=9929, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit be705e7bdfb9c2e8e72841deb91544a06f6f1c99.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3887](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13855719086/1/nebius-x86-64/summary/ya-test.html) | [3887](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13855719086/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T11:09:33Z
body: <!-- status pr=3187, run=9964, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 3eb347f1e7cf25ea73faa9aca0fa055c06de6cb7.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13896262127/1/nebius-x86-64/summary/ya-test.html) | [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13896262127/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T13:47:21Z
body: <!-- status pr=3187, run=9979, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 66289d87ba0fcebf098ce0095fd99c7afef2a479.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13899321689/1/nebius-x86-64/summary/ya-test.html) | [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13899321689/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-19T07:39:06Z
body: <!-- status pr=3187, run=10038, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit f2d9d356756b17c3da53581d017da437a6a6971a.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13940075965/1/nebius-x86-64/summary/ya-test.html) | [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13940075965/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-20T10:24:45Z
body: <!-- status pr=3187, run=10106, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit e4628d014b6202fb2992904335a07b00f1cfcc06.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13965369025/1/nebius-x86-64/summary/ya-test.html) | [3887](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13965369025/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [1](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13965369025/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-20T12:58:04Z
body: <!-- status pr=3187, run=10111, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit df056051fd70c3faafe5b97ac1507738b468d4a7.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13968230969/1/nebius-x86-64/summary/ya-test.html) | [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/13968230969/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-24T12:46:41Z
body: <!-- status pr=3187, run=10194, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 3a73d297462a32c251b04adcf86c23d7e0a3c71c.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14033943618/1/nebius-x86-64/summary/ya-test.html) | [3884](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14033943618/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [4](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14033943618/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-25T11:07:57Z
body: <!-- status pr=3187, run=10244, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 39032294ced1f8ca95c9456e2ee4d5ae9b43c8af.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14056367032/1/nebius-x86-64/summary/ya-test.html) | [3888](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14056367032/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-25T13:33:18Z
body: <!-- status pr=3187, run=10258, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 180d5484a615dac0adec3921830b32ac16e8c592.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3910](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14059058819/1/nebius-x86-64/summary/ya-test.html) | [3910](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14059058819/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-03T13:07:33Z
body: <!-- status pr=3187, run=10588, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:red_circle: **linux-x86_64-relwithdebinfo**: some tests FAILED for commit 4871f9643984c820b69d1d38b0a4ec02489c083a.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3910](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14241603042/1/nebius-x86-64/summary/ya-test.html) | [3905](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14241603042/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | [5](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14241603042/1/nebius-x86-64/summary/ya-test.html#FAIL) | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-04T08:21:40Z
body: <!-- status pr=3187, run=10612, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit dbdd219924f95a77bff361e839d539a3db7eb849.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3910](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14259512285/1/nebius-x86-64/summary/ya-test.html) | [3910](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14259512285/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-10T17:06:20Z
body: <!-- status pr=3187, run=10769, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit e173a9506a6f8ba888ff9728a36c13cbdc284291.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3978](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14383496269/1/nebius-x86-64/summary/ya-test.html) | [3978](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14383496269/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-21T09:36:28Z
body: <!-- status pr=3187, run=10964, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 2fe7ede1f1ea455a73725782ae741227ed48e6db.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3996](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14569441402/1/nebius-x86-64/summary/ya-test.html) | [3996](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14569441402/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-22T10:29:26Z
body: <!-- status pr=3187, run=10992, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit 2ff7e5f7ede49af1f7f2f0609bdde27211a08dd5.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3996](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14590117408/1/nebius-x86-64/summary/ya-test.html) | [3996](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14590117408/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-23T09:23:47Z
body: <!-- status pr=3187, run=11017, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit af29d05214e705574020656bf71afd440ae77ca1.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3997](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14612234998/1/nebius-x86-64/summary/ya-test.html) | [3997](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14612234998/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-23T17:13:06Z
body: <!-- status pr=3187, run=11038, build_preset=linux-x86_64-relwithdebinfo, dry_run=False -->
> [!NOTE]
> This is an automated comment that will be appended during run.

:green_circle: **linux-x86_64-relwithdebinfo**: all tests PASSED for commit d02d975155912da798c98c020f228245986c92a0.
| TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED<sup>[?](https://github.com/ydb-platform/nbs/tree/main/.github/config "All mute rules are defined here")</sup> |
| ---: | ---: | ---: | ---: | ---: | ---: |
| [3998](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14621712270/1/nebius-x86-64/summary/ya-test.html) | [3998](https://github-actions-s3.storage.eu-north1.nebius.cloud/ydb-platform/nbs/PR-check/14621712270/1/nebius-x86-64/summary/ya-test.html#PASS) | 0 | 0 | 0 | 0 |
author_association: CONTRIBUTOR