Repository: apache/ozone
Pull request #7985: HDDS-12428. Avoid force closing OPEN/CLOSING replica of a CLOSED Container
Author: login: swamirishi
Created: 2025-02-28T03:38:15Z | Merged: 2025-03-06T17:30:08Z

## What changes were proposed in this pull request?
If the container is in CLOSED state and the Replica state in SCM is OPEN/CLOSING state, we are force closing the Container without comparing the BCSID. We should not force close the replicas in OPEN/CLOSING state.
Changing the should close function to return the expected final state and setting force parameter based on the value returned.

## What is the link to the Apache JIRA
https://issues.apache.org/jira/browse/HDDS-12428?jql=text%20~%20%22force%20close%22

## How was this patch tested?
Adding unit tests.



Diff:
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
index 97fece928b34..5fde4659829b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
@@ -57,8 +57,13 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
@@ -66,24 +71,14 @@ public boolean handle(ContainerCheckRequest request) {
     }
     LOG.debug("Checking container {} in MismatchedReplicasHandler",
         containerInfo);
-
-    if (request.isReadOnly()) {
-      return false;
-    }
     // close replica if needed
     for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
+      ContainerReplicaProto.State replicaState = getTransitionState(containerInfo, replica);
+      if (replicaState != null) {
         LOG.debug("Sending close command for mismatched replica {} of " +
             "container {}.", replica, containerInfo);
-
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
+        replicationManager.sendCloseContainerReplicaCommand(
+            containerInfo, replica.getDatanodeDetails(), ContainerReplicaProto.State.CLOSED.equals(replicaState));
       }
     }
 
@@ -95,23 +90,24 @@ public boolean handle(ContainerCheckRequest request) {
   }
 
   /**
-   * If a CLOSED or QUASI-CLOSED container has an OPEN or CLOSING replica,
-   * there is a state mismatch. QUASI_CLOSED replica of a CLOSED container
-   * should be closed if their sequence IDs match.
+   * Returns the final expected closed state type based on the scm container state and the replica state.
    * @param replica replica to check for mismatch and if it should be closed
-   * @return true if the replica should be closed, else false
+   * @return null if the replica should not be closed, else CLOSED/QUASI_CLOSED based on the replica's
+   * state.
    */
-  private boolean shouldBeClosed(ContainerInfo container,
-      ContainerReplica replica) {
+  private ContainerReplicaProto.State getTransitionState(ContainerInfo container,
+                                                         ContainerReplica replica) {
     if (replica.getState() == ContainerReplicaProto.State.OPEN ||
         replica.getState() == ContainerReplicaProto.State.CLOSING) {
-      return true;
+      return HddsProtos.ReplicationType.RATIS == container.getReplicationType() ?
+          ContainerReplicaProto.State.QUASI_CLOSED : ContainerReplicaProto.State.CLOSED;
     }
 
     // a quasi closed replica of a closed container should be closed if their
     // sequence IDs match
     return container.getState() == HddsProtos.LifeCycleState.CLOSED &&
         replica.getState() == ContainerReplicaProto.State.QUASI_CLOSED &&
-        container.getSequenceId() == replica.getSequenceId();
+        container.getSequenceId() == replica.getSequenceId() ? ContainerReplicaProto.State.CLOSED
+        : null;
   }
 }
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java
index 769007546b6b..d8b660c9a3c1 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java
@@ -223,12 +223,12 @@ public void testCloseCommandSentForMismatchedRatisReplicas() {
     assertFalse(handler.handle(readRequest));
 
     verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
-        containerInfo, mismatch1.getDatanodeDetails(), true);
+        containerInfo, mismatch1.getDatanodeDetails(), false);
     verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
-        containerInfo, mismatch2.getDatanodeDetails(), true);
+        containerInfo, mismatch2.getDatanodeDetails(), false);
     // close command should not be sent for unhealthy replica
     verify(replicationManager, times(0)).sendCloseContainerReplicaCommand(
-        containerInfo, mismatch3.getDatanodeDetails(), true);
+        containerInfo, mismatch3.getDatanodeDetails(), false);
   }
 
   /**
@@ -326,4 +326,59 @@ public void testQuasiClosedReplicaOfClosedContainer() {
     verify(replicationManager, times(0)).sendCloseContainerReplicaCommand(containerInfo,
         differentSeqID.getDatanodeDetails(), true);
   }
+
+  @Test
+  public void testCloseCommandSentForMismatchedRatisReplicasWithIncorrectBCSID() {
+    ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+        ratisReplicationConfig, 1, CLOSED, 1000);
+    ContainerReplica mismatch1 = ReplicationTestUtil.createContainerReplica(
+        containerInfo.containerID(), 0,
+        HddsProtos.NodeOperationalState.IN_SERVICE,
+        ContainerReplicaProto.State.OPEN, 99);
+    ContainerReplica mismatch2 = ReplicationTestUtil.createContainerReplica(
+        containerInfo.containerID(), 0,
+        HddsProtos.NodeOperationalState.IN_SERVICE,
+        ContainerReplicaProto.State.CLOSING, 999);
+    ContainerReplica mismatch3 = ReplicationTestUtil.createContainerReplica(
+        containerInfo.containerID(), 0,
+        HddsProtos.NodeOperationalState.IN_SERVICE,
+        ContainerReplicaProto.State.QUASI_CLOSED, 1000);
+    ContainerReplica mismatch4 = ReplicationTestUtil.createContainerReplica(
+        containerInfo.containerID(), 0,
+        HddsProtos.NodeOperationalState.IN_SERVICE,
+        ContainerReplicaProto.State.CLOSING, 1000);
+    Set<ContainerReplica> containerReplicas = new HashSet<>();
+    containerReplicas.add(mismatch1);
+    containerReplicas.add(mismatch2);
+    containerReplicas.add(mismatch3);
+    containerReplicas.add(mismatch4);
+    ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+        .setPendingOps(Collections.emptyList())
+        .setReport(new ReplicationManagerReport())
+        .setContainerInfo(containerInfo)
+        .setContainerReplicas(containerReplicas)
+        .build();
+    ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder()
+        .setPendingOps(Collections.emptyList())
+        .setReport(new ReplicationManagerReport())
+        .setContainerInfo(containerInfo)
+        .setContainerReplicas(containerReplicas)
+        .setReadOnly(true)
+        .build();
+
+    // this handler always returns false so other handlers can fix issues
+    // such as under replication
+    assertFalse(handler.handle(request));
+    assertFalse(handler.handle(readRequest));
+
+    verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
+        containerInfo, mismatch1.getDatanodeDetails(), false);
+    verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
+        containerInfo, mismatch2.getDatanodeDetails(), false);
+    // close command should not be sent for unhealthy replica
+    verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
+        containerInfo, mismatch3.getDatanodeDetails(), true);
+    verify(replicationManager, times(1)).sendCloseContainerReplicaCommand(
+        containerInfo, mismatch4.getDatanodeDetails(), false);
+  }
 }


Reviews:
user: login: siddhantsangwan
body: Looks good, just a minor comment. Let's continue to test this.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T06:28:03Z
user: login: sumitagrawl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T06:58:46Z
user: login: nandakumar131
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T09:52:50Z
user: login: siddhantsangwan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T09:58:49Z
user: login: nandakumar131
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T12:43:01Z
user: login: siddhantsangwan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T13:16:11Z
user: login: swamirishi
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T17:06:46Z
user: login: errose28
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-01T06:05:53Z
user: login: swamirishi
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-01T13:02:53Z
user: login: nandakumar131
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-01T18:25:24Z
user: login: sumitagrawl
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T16:27:26Z

Review comments:
diff_hunk: @@ -95,23 +89,23 @@ public boolean handle(ContainerCheckRequest request) {
   }
 
   /**
-   * If a CLOSED or QUASI-CLOSED container has an OPEN or CLOSING replica,
-   * there is a state mismatch. QUASI_CLOSED replica of a CLOSED container
-   * should be closed if their sequence IDs match.
+   * Returns the final expected closed state type based on the scm container state and the replica state.
    * @param replica replica to check for mismatch and if it should be closed
-   * @return true if the replica should be closed, else false
+   * @return non null closed state if the replica should be closed, else CLOSED/QUASI_CLOSED based on the replica's
+   * state.
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: siddhantsangwan
body: ```suggestion
   * @return null if the replica should not be closed, else CLOSED/QUASI_CLOSED based on the replica's
   * state.
```
created_at: 2025-02-28T06:27:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: sumitagrawl
body: We can have info log for sending close in this handle, including force, replica state and container state. This will be useful for debugging.
created_at: 2025-02-28T06:58:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: nandakumar131
body: `replicationManager.sendCloseContainerReplicaCommand` ends up calling `ReplicationManager#sendDatanodeCommand` which already logs the command details (including force flag) and the ContainerInfo at `info` level.

Adding another info level log here will be redundant.

Here at debug level, the additional information that's getting logged is `replica` info.
created_at: 2025-02-28T09:52:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: siddhantsangwan
body: So just to be clear, we're also sending a force close if the replica is open/closing and the bcsid matches. I think this is different from the previous commit's behaviour.
created_at: 2025-02-28T09:58:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: nandakumar131
body: Yes, if the BCSID matches we are sending force close command to that Container replica.

This is not different from the previous behaviour, we were sending force close for open/closing replica even without checking the sequence Id. With this change we will only send force close if the sequence Id matches.

We believe  that the BCSID of a CLOSED container is the highest (correct).
created_at: 2025-02-28T12:43:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: siddhantsangwan
body: No, I meant it's different from the behaviour that @swamirishi had pushed in his commit. Your latest one looks fine to me, but @swamirishi should take a look.
created_at: 2025-02-28T13:16:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: swamirishi
body: @nandakumar131 Lets not send a force close directly from OPEN/CLOSING. Let us move it to QUASI_CLOSED and let the QUASI_CLOSED container handler do it. Let us not have duplicate coding logic floating around.
created_at: 2025-02-28T17:06:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: errose28
body: We should look at this in context with the [datanode flow](https://github.com/apache/ozone/blob/39437eae0cbc4de6078ca16f80d5c579d5a93fbf/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java#L116). A non-force close to an open or closing replica will still try to use the pipeline if it is available. Only if that fails on the DN will it quasi-close the container. Then we can invoke the force flow from SCM. Since we are checking the BCSID to decide whether to force close or not, we don't necessarily need the pipeline though.

I think both approaches give the correct result. However I think it makes sense to at least try a non-force close in case the pipeline is still there and let the DN tell us if force is required by moving the replica to quasi-closed. Otherwise it feels like we skipped a step.
created_at: 2025-03-01T06:05:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: swamirishi
body: yeah i agree with you @errose28 
created_at: 2025-03-01T13:02:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,35 +58,28 @@ public MismatchedReplicasHandler(
    */
   @Override
   public boolean handle(ContainerCheckRequest request) {
-    ContainerInfo containerInfo = request.getContainerInfo();
-    Set<ContainerReplica> replicas = request.getContainerReplicas();
+    if (request.isReadOnly()) {
+      return false;
+    }
+
+    final ContainerInfo containerInfo = request.getContainerInfo();
+    final Set<ContainerReplica> replicas = request.getContainerReplicas();
+
     if (containerInfo.getState() != HddsProtos.LifeCycleState.CLOSED &&
         containerInfo.getState() != HddsProtos.LifeCycleState.QUASI_CLOSED) {
       // Handler is only relevant for CLOSED or QUASI-CLOSED containers.
       return false;
     }
-    LOG.debug("Checking container {} in MismatchedReplicasHandler",
-        containerInfo);
 
-    if (request.isReadOnly()) {
-      return false;
-    }
-    // close replica if needed
-    for (ContainerReplica replica : replicas) {
-      if (shouldBeClosed(containerInfo, replica)) {
-        LOG.debug("Sending close command for mismatched replica {} of " +
-            "container {}.", replica, containerInfo);
+    LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo);
 
-        if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), true);
-        } else if (containerInfo.getState() ==
-            HddsProtos.LifeCycleState.QUASI_CLOSED) {
-          replicationManager.sendCloseContainerReplicaCommand(
-              containerInfo, replica.getDatanodeDetails(), false);
-        }
-      }
-    }
+    final Predicate<ContainerReplica> shouldSendClose = (r) -> shouldSendClose(containerInfo, r);
+
+    replicas.stream().filter(shouldSendClose).forEach(r -> {
+      LOG.debug("Sending close command for mismatched replica {} of container {}.", r, containerInfo);
+      replicationManager.sendCloseContainerReplicaCommand(
+          containerInfo, r.getDatanodeDetails(), shouldForceClose(containerInfo, r));
path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
user: login: nandakumar131
body: The same handler is used for EC Containers as well and for EC Containers the Datanode [expects force flag to be set](https://github.com/apache/ozone/blob/39437eae0cbc4de6078ca16f80d5c579d5a93fbf/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java#L128) for Replicas in OPEN/CLOSING state.

We should add logic based on Container type.

If everyone feels strongly about this, I can change the behaviour for Ratis Containers.


created_at: 2025-03-01T18:25:24Z
author_association: CONTRIBUTOR

PR comments:
user: login: swamirishi
created_at: 2025-03-06T11:06:33Z
body: @nandakumar131 Can you take a look at this. If good we can merge this
author_association: CONTRIBUTOR
user: login: nandakumar131
created_at: 2025-03-06T17:28:55Z
body: Looks good!
author_association: CONTRIBUTOR
user: login: nandakumar131
created_at: 2025-03-06T17:30:47Z
body: Thanks @swamirishi for the contribution and @siddhantsangwan @sumitagrawl @errose28 for the review.
author_association: CONTRIBUTOR