Repository: cockroachdb/cockroach
Pull request #140979: kvserver: report store-liveness unreachable replicas
Author: login: pav-kv
Labels: name: v25.2.0-prerelease
Created: 2025-02-10T14:45:35Z | Merged: 2025-03-06T17:05:33Z

This commit triggers `addUnreachableRemoteReplica` when the corresponding store liveness is disconnected. Eventually, this triggers `ReportUnreachable` call to raft, which transfers the flow to `StateProbe`.

Epic: none
Release note: none


Diff:
diff --git a/pkg/kv/kvserver/client_raft_test.go b/pkg/kv/kvserver/client_raft_test.go
index d76f07c703a9..9a85f70a3304 100644
--- a/pkg/kv/kvserver/client_raft_test.go
+++ b/pkg/kv/kvserver/client_raft_test.go
@@ -38,6 +38,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader"
 	raft "github.com/cockroachdb/cockroach/pkg/raft"
 	"github.com/cockroachdb/cockroach/pkg/raft/raftpb"
+	"github.com/cockroachdb/cockroach/pkg/raft/tracker"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
@@ -2721,7 +2722,7 @@ func TestWedgedReplicaDetection(t *testing.T) {
 			if len(lastUpdateTimes) == 3 {
 				return nil
 			}
-			return errors.Errorf("expected leader replica to have 3 entries in lastUpdateTimes map, found %s", lastUpdateTimes)
+			return errors.Errorf("expected leader replica to have 3 entries in lastUpdateTimes map, found %v", lastUpdateTimes)
 		})
 
 		// Lock the follower replica to prevent it from making progress from now
@@ -2761,7 +2762,7 @@ func TestWedgedReplicaDetection(t *testing.T) {
 		leaderNow := leaderClock.PhysicalTime()
 		if !leaderRepl.IsFollowerActiveSince(followerID, leaderNow, inactivityThreshold) {
 			t.Fatalf("expected follower to still be considered active; "+
-				"follower id: %d, last update times: %s, leader clock: %s",
+				"follower id: %d, last update times: %v, leader clock: %s",
 				followerID, leaderRepl.LastUpdateTimes(), leaderNow)
 		}
 
@@ -2783,7 +2784,7 @@ func TestWedgedReplicaDetection(t *testing.T) {
 			leaderNow = leaderClock.PhysicalTime()
 			if leaderRepl.IsFollowerActiveSince(followerID, leaderNow, inactivityThreshold) {
 				return errors.Errorf("expected follower to be considered inactive; "+
-					"follower id: %d, last update times: %s, leader clock: %s",
+					"follower id: %d, last update times: %v, leader clock: %s",
 					followerID, leaderRepl.LastUpdateTimes(), leaderNow)
 			}
 			return nil
@@ -2887,6 +2888,64 @@ func TestReportUnreachableHeartbeats(t *testing.T) {
 	}
 }
 
+// TestReportUnreachableStoreLiveness tests that if a follower's store liveness
+// is expired, ReportUnreachable is called and the corresponding raft flow is
+// transferred to StateProbe.
+func TestReportUnreachableStoreLiveness(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	// Enable leader fortification, store liveness, and leader leases.
+	ctx := context.Background()
+	st := cluster.MakeTestingClusterSettings()
+	kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseLeader)
+
+	tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{
+		ReplicationMode: base.ReplicationManual,
+		ServerArgs:      base.TestServerArgs{Settings: st},
+	})
+	defer tc.Stopper().Stop(ctx)
+
+	key := tc.ScratchRange(t)
+	tc.AddVotersOrFatal(t, key, tc.Targets(1, 2)...)
+	desc := tc.LookupRangeOrFatal(t, key)
+
+	// Send a write, to get things going.
+	s1 := tc.GetFirstStoreFromServer(t, 0)
+	incArgs := incrementArgs(key, 5)
+	if _, err := kv.SendWrapped(ctx, s1.TestSender(), incArgs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Partition n3 from other nodes (including the leader on n1).
+	partRange, err := setupPartitionedRange(tc, desc.RangeID, 0, 2,
+		true /* activated */, unreliableRaftHandlerFuncs{})
+	require.NoError(t, err)
+
+	leaderRepl := s1.LookupReplica(roachpb.RKey(key))
+	require.NotNil(t, leaderRepl)
+	s3Repl := tc.GetFirstStoreFromServer(t, 2).LookupReplica(roachpb.RKey(key))
+	require.NotNil(t, s3Repl)
+
+	waitForFlowState := func(state tracker.StateType) {
+		t.Helper()
+		testutils.SucceedsSoon(t, func() error {
+			pr, ok := leaderRepl.RaftStatus().Progress[raftpb.PeerID(s3Repl.ReplicaID())]
+			require.True(t, ok)
+			if pr.State != state {
+				return errors.Newf("raft flow in %v, want %v", pr.State, state)
+			}
+			return nil
+		})
+	}
+
+	// Eventually, the s3 store liveness expires, and the flow transfers to StateProbe.
+	waitForFlowState(tracker.StateProbe)
+	// After healing the partition, the flow returns back to StateReplicate.
+	partRange.deactivate()
+	waitForFlowState(tracker.StateReplicate)
+}
+
 // TestReportUnreachableRemoveRace adds and removes the raft leader replica
 // repeatedly while one of its peers is unreachable in an attempt to expose
 // races (primarily in asynchronous coalesced heartbeats).
diff --git a/pkg/kv/kvserver/helpers_test.go b/pkg/kv/kvserver/helpers_test.go
index 65ed71b4f984..adb0153ce52c 100644
--- a/pkg/kv/kvserver/helpers_test.go
+++ b/pkg/kv/kvserver/helpers_test.go
@@ -415,7 +415,7 @@ func (r *Replica) NumPendingProposals() int64 {
 	return r.numPendingProposalsRLocked()
 }
 
-func (r *Replica) LastUpdateTimes() map[roachpb.ReplicaID]time.Time {
+func (r *Replica) LastUpdateTimes() map[roachpb.ReplicaID]lastReplicaUpdateTime {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 	return maps.Clone(r.mu.lastUpdateTimes)
diff --git a/pkg/kv/kvserver/raft_log_queue_test.go b/pkg/kv/kvserver/raft_log_queue_test.go
index b5d461fe10f6..c01c69256812 100644
--- a/pkg/kv/kvserver/raft_log_queue_test.go
+++ b/pkg/kv/kvserver/raft_log_queue_test.go
@@ -370,10 +370,10 @@ func TestUpdateRaftStatusActivity(t *testing.T) {
 		{
 			replicas: []roachpb.ReplicaDescriptor{{ReplicaID: 1}, {ReplicaID: 2}, {ReplicaID: 3}},
 			prs:      []tracker.Progress{{RecentActive: false}, {RecentActive: true}},
-			lastUpdate: map[roachpb.ReplicaID]time.Time{
-				1: now.Add(-1 * inactivityThreashold / 2),
-				2: now.Add(-1 - inactivityThreashold),
-				3: now,
+			lastUpdate: map[roachpb.ReplicaID]lastReplicaUpdateTime{
+				1: {update: now.Add(-1 * inactivityThreashold / 2)},
+				2: {update: now.Add(-1 - inactivityThreashold)},
+				3: {update: now},
 			},
 			now: now,
 
diff --git a/pkg/kv/kvserver/replica_proposal_quota.go b/pkg/kv/kvserver/replica_proposal_quota.go
index 4b8ca667b13e..0dc4d75d244b 100644
--- a/pkg/kv/kvserver/replica_proposal_quota.go
+++ b/pkg/kv/kvserver/replica_proposal_quota.go
@@ -147,7 +147,7 @@ func (r *Replica) updateProposalQuotaRaftMuLocked(
 	shouldInitQuotaPool := false
 	if r.mu.leaderID != lastLeaderID {
 		if r.replicaID == r.mu.leaderID {
-			r.mu.lastUpdateTimes = make(map[roachpb.ReplicaID]time.Time)
+			r.mu.lastUpdateTimes = make(map[roachpb.ReplicaID]lastReplicaUpdateTime)
 			r.mu.lastUpdateTimes.updateOnBecomeLeader(r.shMu.state.Desc.Replicas().Descriptors(), now)
 			r.mu.replicaFlowControlIntegration.onBecameLeader(ctx)
 			r.mu.lastProposalAtTicks = r.mu.ticks // delay imminent quiescence
diff --git a/pkg/kv/kvserver/replica_raft.go b/pkg/kv/kvserver/replica_raft.go
index 62b1c130e300..ab8bd7011566 100644
--- a/pkg/kv/kvserver/replica_raft.go
+++ b/pkg/kv/kvserver/replica_raft.go
@@ -1525,7 +1525,7 @@ func (r *Replica) tick(
 		r.mu.lastUpdateTimes.update(r.replicaID, r.Clock().PhysicalTime())
 		// We also update lastUpdateTimes for replicas that provide store liveness
 		// support to the leader.
-		r.updateLastUpdateTimesUsingStoreLivenessRLocked(storeClockTimestamp)
+		r.updateLastUpdateTimesUsingStoreLivenessLocked(storeClockTimestamp)
 	}
 
 	r.mu.ticks++
@@ -2774,16 +2774,36 @@ func shouldTransferRaftLeadershipToLeaseholderLocked(
 	return lhCaughtUp
 }
 
+type lastReplicaUpdateTime struct {
+	update      time.Time
+	unreachable bool
+}
+
 // a lastUpdateTimesMap is maintained on the Raft leader to keep track of the
 // last communication received from followers, which in turn informs the quota
 // pool and log truncations.
-type lastUpdateTimesMap map[roachpb.ReplicaID]time.Time
+type lastUpdateTimesMap map[roachpb.ReplicaID]lastReplicaUpdateTime
 
 func (m lastUpdateTimesMap) update(replicaID roachpb.ReplicaID, now time.Time) {
 	if m == nil {
 		return
 	}
-	m[replicaID] = now
+	m[replicaID] = lastReplicaUpdateTime{update: now, unreachable: false}
+}
+
+// updateUnreachable marks the given replica ID as unreachable. Returns false
+// iff it is already marked as unreachable.
+func (m lastUpdateTimesMap) updateUnreachable(replicaID roachpb.ReplicaID) bool {
+	if m == nil {
+		return false
+	}
+	last := m[replicaID]
+	if last.unreachable {
+		return false
+	}
+	last.unreachable = true
+	m[replicaID] = last
+	return true
 }
 
 // updateOnUnquiesce is called when the leader unquiesces. In that case, we
@@ -2828,7 +2848,7 @@ func (m lastUpdateTimesMap) isFollowerActiveSince(
 		// replicas were updated).
 		return false
 	}
-	return now.Sub(lastUpdateTime) <= threshold
+	return now.Sub(lastUpdateTime.update) <= threshold
 }
 
 // maybeAcquireSnapshotMergeLock checks whether the incoming snapshot subsumes
@@ -3098,8 +3118,8 @@ func (r *Replica) printRaftTail(
 // under the raft fortification protocol, where failure detection is subsumed by
 // store liveness.
 //
-// This method assume that Replica.mu is held in read mode.
-func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
+// This method assumes that Replica.mu is locked for writes.
+func (r *Replica) updateLastUpdateTimesUsingStoreLivenessLocked(
 	storeClockTimestamp hlc.ClockTimestamp,
 ) {
 	// If store liveness is not enabled, there is nothing to do. The
@@ -3112,9 +3132,19 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if r.mu.lastUpdateTimes.updateUnreachable(desc.ReplicaID) {
+			r.addUnreachableRemoteReplica(desc.ReplicaID)
 		}
 	}
 }
diff --git a/pkg/kv/kvserver/replica_raft_test.go b/pkg/kv/kvserver/replica_raft_test.go
index abec22ccb04c..cb04dedd0735 100644
--- a/pkg/kv/kvserver/replica_raft_test.go
+++ b/pkg/kv/kvserver/replica_raft_test.go
@@ -51,12 +51,24 @@ func TestLastUpdateTimesMap(t *testing.T) {
 	t2 := t1.Add(time.Second)
 	m.update(3, t1)
 	m.update(1, t2)
-	assert.EqualValues(t, map[roachpb.ReplicaID]time.Time{1: t2, 3: t1}, m)
-	descs := []roachpb.ReplicaDescriptor{{ReplicaID: 1}, {ReplicaID: 2}, {ReplicaID: 3}, {ReplicaID: 4}}
 
+	upd := func(t time.Time, unreachable bool) lastReplicaUpdateTime {
+		return lastReplicaUpdateTime{update: t, unreachable: unreachable}
+	}
+	assert.EqualValues(t, map[roachpb.ReplicaID]lastReplicaUpdateTime{
+		1: upd(t2, false),
+		3: upd(t1, false),
+	}, m)
+
+	descs := []roachpb.ReplicaDescriptor{{ReplicaID: 1}, {ReplicaID: 2}, {ReplicaID: 3}, {ReplicaID: 4}}
 	t3 := t2.Add(time.Second)
 	m.updateOnBecomeLeader(descs, t3)
-	assert.EqualValues(t, map[roachpb.ReplicaID]time.Time{1: t3, 2: t3, 3: t3, 4: t3}, m)
+	assert.EqualValues(t, map[roachpb.ReplicaID]lastReplicaUpdateTime{
+		1: upd(t3, false),
+		2: upd(t3, false),
+		3: upd(t3, false),
+		4: upd(t3, false),
+	}, m)
 
 	t4 := t3.Add(time.Second)
 	descs = append(descs, []roachpb.ReplicaDescriptor{{ReplicaID: 5}, {ReplicaID: 6}}...)
@@ -70,13 +82,20 @@ func TestLastUpdateTimesMap(t *testing.T) {
 		7: {State: tracker.StateReplicate}, // ignored, not in descs
 	}
 	m.updateOnUnquiesce(descs, prs, t4)
-	assert.EqualValues(t, map[roachpb.ReplicaID]time.Time{
-		1: t4,
-		2: t3,
-		3: t3,
-		4: t3,
-		6: t4,
+	assert.EqualValues(t, map[roachpb.ReplicaID]lastReplicaUpdateTime{
+		1: upd(t4, false),
+		2: upd(t3, false),
+		3: upd(t3, false),
+		4: upd(t3, false),
+		6: upd(t4, false),
 	}, m)
+
+	assert.True(t, m.updateUnreachable(4))
+	assert.False(t, m.updateUnreachable(4))
+	assert.Equal(t, upd(t3, true), m[4])
+	m.update(4, t4)
+	assert.Equal(t, upd(t4, false), m[4])
+	assert.True(t, m.updateUnreachable(4))
 }
 
 func Test_handleRaftReadyStats_SafeFormat(t *testing.T) {


Reviews:
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-02-10T14:50:49Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-02-10T15:34:16Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-02-10T16:02:03Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-02-10T16:07:24Z
user: login: miraradeva
body: Is there a good way to test this new transition?

LGTM! I'll defer the approval to @iskettaneh (who worked on this map recently) and @sumeerbhola. 
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-10T17:55:58Z
user: login: iskettaneh
body: <img class="emoji" title=":lgtm:" alt=":lgtm:" align="absmiddle" src="https://reviewable.io/lgtm.png" height="20" width="61"/> from my end.
+1 to what Mira suggested of testing this new transition.

> Reviewed 5 of 5 files at r2, 5 of 5 files at r3, all commit messages.
> *[Reviewable](https://reviewable.io/reviews/cockroachdb/cockroach/140979#-:-OIlINvRDM7H0uxl591k:b-ml1osb)* status: <span>:shipit:</span> complete! 1 of 0 LGTMs obtained (waiting on @pav-kv and @sumeerbhola)



<!-- Sent from Reviewable.io -->

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-02-10T19:23:21Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-02-10T20:09:03Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T13:41:37Z
user: login: miraradeva
body: <img class="emoji" title=":lgtm:" alt=":lgtm:" align="absmiddle" src="https://reviewable.io/lgtm.png" height="20" width="61"/> Thanks for adding the test!

> Reviewed 5 of 5 files at r4, 5 of 5 files at r7, 6 of 6 files at r8, 1 of 1 files at r9, all commit messages.
> *[Reviewable](https://reviewable.io/reviews/cockroachdb/cockroach/140979#-:-OKgGMW38lPokKNhuxSn:b-pdmsbk)* status: <span>:shipit:</span> complete! 1 of 0 LGTMs obtained (and 1 stale) (waiting on @pav-kv and @sumeerbhola)



<!-- Sent from Reviewable.io -->

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T16:24:16Z
user: login: pav-kv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-10T15:50:19Z

Review comments:
diff_hunk: @@ -2753,16 +2753,21 @@ func shouldTransferRaftLeadershipToLeaseholderLocked(
 	return lhCaughtUp
 }
 
+type lastReplicaUpdateTime struct {
+	update      time.Time
+	unreachable bool
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Needed to add this field so that `addUnreachableRemoteReplica` / `ReportUnreachable` are called only once per disconnect. Otherwise, these are triggered every tick from `updateLastUpdateTimesUsingStoreLivenessRLocked`.
created_at: 2025-02-10T14:50:48Z
author_association: COLLABORATOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
+			r.mu.lastUpdateTimes[desc.ReplicaID] = upd
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: This update might be problematic because `Replica.mu` is only locked for reads.
created_at: 2025-02-10T15:34:16Z
author_association: COLLABORATOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
+			r.mu.lastUpdateTimes[desc.ReplicaID] = upd
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Btw, the above `r.mu.lastUpdateTimes.update` is incorrect for the same reason.
created_at: 2025-02-10T16:02:03Z
author_association: COLLABORATOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
+			r.mu.lastUpdateTimes[desc.ReplicaID] = upd
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Luckily, `mu` is already locked for writes. So I just "fixed" this by renaming the method in another commit.
created_at: 2025-02-10T16:07:24Z
author_association: COLLABORATOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
path: pkg/kv/kvserver/replica_raft.go
user: login: miraradeva
body: This call to `SupportFrom` happens only when a leader checks if it has support from a follower. So the leader will mark the follower as unreachable, but not vice versa. Presumably that's ok for the call to `r.addUnreachableRemoteReplica` because the flow control streams are always managed by the leader?
created_at: 2025-02-10T17:52:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
path: pkg/kv/kvserver/replica_raft.go
user: login: miraradeva
body: nit: maybe we can have two update functions for `lastUpdateTimes`:

- `updateTime(replicaID roachpb.ReplicaID, now time.Time)`
- `updateUnreachable(replicaID roachpb.ReplicaID, unreachable bool)`
created_at: 2025-02-10T17:53:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
+			r.mu.lastUpdateTimes[desc.ReplicaID] = upd
path: pkg/kv/kvserver/replica_raft.go
user: login: iskettaneh
body: Good catch!
<!-- Reviewable comment -OIlHNUi4ItypsS22t03:b6zrrk4 -->
<!-- Sent from Reviewable.io -->

created_at: 2025-02-10T19:23:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Yes.
created_at: 2025-02-10T20:09:02Z
author_association: COLLABORATOR
diff_hunk: @@ -3091,9 +3096,21 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if upd := r.mu.lastUpdateTimes[desc.ReplicaID]; !upd.unreachable {
+			upd.unreachable = true
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Good idea. Done.
created_at: 2025-03-06T13:41:37Z
author_association: COLLABORATOR
diff_hunk: @@ -3112,9 +3132,19 @@ func (r *Replica) updateLastUpdateTimesUsingStoreLivenessRLocked(
 	for _, desc := range r.descRLocked().Replicas().Descriptors() {
 		// If the replica's store if providing store liveness support, update
 		// lastUpdateTimes to indicate that it is alive.
+		//
+		// Otherwise, mark the replica as unreachable, so that eventually raft is
+		// notified by ReportUnreachable, and the flow transfers to StateProbe. Do
+		// this only the first time, so that addUnreachableRemoteReplica is called
+		// only once per disconnect.
+		//
+		// TODO(pav-kv): there is an overlap between r.mu.lastUpdateTimes and
+		// r.unreachablesMu. Unify them into one mechanism.
 		_, curExp := (*replicaRLockedStoreLiveness)(r).SupportFrom(raftpb.PeerID(desc.ReplicaID))
 		if storeClockTimestamp.ToTimestamp().LessEq(curExp) {
 			r.mu.lastUpdateTimes.update(desc.ReplicaID, r.Clock().PhysicalTime())
+		} else if r.mu.lastUpdateTimes.updateUnreachable(desc.ReplicaID) {
path: pkg/kv/kvserver/replica_raft.go
user: login: pav-kv
body: Wonder if there is a bug here: what if the `storeClockTimestamp` is zero / unset? We're going to unconditionally report replica as unavailable.
created_at: 2025-03-10T15:50:19Z
author_association: COLLABORATOR

PR comments:
user: login: cockroach-teamcity
created_at: 2025-02-10T14:45:46Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/140979)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: pav-kv
created_at: 2025-03-06T13:33:00Z
body: @miraradeva @iskettaneh Added a test. Verified that it fails before the change.
author_association: COLLABORATOR
user: login: pav-kv
created_at: 2025-03-06T16:32:01Z
body: TFTRs!

bors r=iskettaneh,miraradeva
author_association: COLLABORATOR
user: login: craig[bot]
created_at: 2025-03-06T17:05:31Z
body: Build succeeded:
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323080733)
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323083035)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323081193)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323082192)
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323079382)
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323081700)
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323082584)
  * local_roachtest_fips
  * [unit_tests](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323083966)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/13703518683/job/38323079857)
author_association: CONTRIBUTOR