Repository: red-hat-storage/ocs-ci
Pull request #11000: [Stretch cluster] Support zone unaware apps for zone shutdown tests
Author: login: mashetty330
Labels: name: Verified
description: Mark when PR was verified and log provided
name: team/e2e
description: E2E team related issues/PRs
name: size/XL
name: lgtm
Created: 2024-12-05T19:00:25Z | Merged: 2025-05-27T15:17:52Z



Diff:
diff --git a/ocs_ci/helpers/helpers.py b/ocs_ci/helpers/helpers.py
index 9554ba0cd22..ee0140df9fd 100644
--- a/ocs_ci/helpers/helpers.py
+++ b/ocs_ci/helpers/helpers.py
@@ -39,6 +39,7 @@
     query_nb_db_psql_version,
 )
 
+from ocs_ci.ocs.node import get_worker_nodes
 from ocs_ci.ocs import constants, defaults, node, ocp, exceptions
 from ocs_ci.ocs.exceptions import (
     CommandFailed,
@@ -6017,3 +6018,118 @@ def set_configmap_log_level_csi_sidecar(value):
     logger.info(f"Setting CSI_SIDECAR log level to: {value}")
     params = f'{{"data": {{"CSI_SIDECAR_LOG_LEVEL": "{value}"}}}}'
     configmap_obj.patch(params=params, format_type="merge")
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-openshift-storage.rbd.csi.ceph.com-nodeplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
+
+    """
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE,
+        namespace=config.ENV_DATA["cluster_namespace"],
+        resource_name=node_name,
+    )
+
+    if not network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        logger.info("Creating NetworkFence")
+        network_fence_dict = templating.load_yaml(constants.NETWORK_FENCE_CRD)
+        network_fence_dict["metadata"]["name"] = node_name
+        network_fence_dict["spec"]["cidrs"][0] = cidr
+        network_fence_obj = create_resource(**network_fence_dict)
+        if network_fence_obj.ocp.get(
+            resource_name=network_fence_obj.name, dont_raise=True
+        ):
+            logger.info(
+                f"NetworkFence {network_fence_obj.name} for node {node_name} created successfully"
+            )
+    else:
+        logger.info(f"Network fence object for {node_name} already exists!")
+    return network_fence_obj
+
+
+def unfence_node(node_name, delete=False):
+    """
+    Un-fence node
+
+    Args:
+        node_name (str): Name of the node
+        delete (bool): If True, delete the network fence object
+
+    """
+
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE, namespace=config.ENV_DATA["cluster_namespace"]
+    )
+
+    if network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        network_fence_obj.patch(
+            resource_name=node_name,
+            params=f'{{"spec":{{"fenceState": "{constants.ACTION_UNFENCE}" }}}}',
+            format_type="merge",
+        )
+        assert (
+            network_fence_obj.get(resource_name=node_name)["spec"]["fenceState"]
+            != constants.ACTION_FENCE
+        ), f"{node_name} is expected to be unfenced but still its fenced"
+        logger.info(f"Unfenced node {node_name} successfully!")
+
+        if delete:
+            network_fence_obj.delete(resource_name=node_name)
+            logger.info(f"Deleted network fence object for node {node_name}")
+    else:
+        logger.info(f"No networkfence found for node {node_name}")
diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py
index 3c4e1507fe6..27502bf34b3 100644
--- a/ocs_ci/ocs/constants.py
+++ b/ocs_ci/ocs/constants.py
@@ -264,6 +264,8 @@
 OCS_OPERATOR_CONFIG_MAP = "ocs-operator-config"
 SERVICE_TYPE_NODEPORT = "NodePort"
 CLUSTERMANAGEMENTADDON = "ClusterManagementAddOn"
+NETWORK_FENCE_CLASS = "NetworkFenceClass"
+NETWORK_FENCE = "NetworkFence"
 
 # Provisioners
 AWS_EFS_PROVISIONER = "openshift.org/aws-efs"
@@ -326,6 +328,9 @@
 RAM = "rss"
 VIRT = "vms"
 
+ODF_NETWORK_FENCE_CLASS = "odf-networkfenceclass"
+CSI_ADDONS_NODE_KIND = "CSIAddonsNode"
+
 # cluster types
 MS_CONSUMER_TYPE = "consumer"
 MS_PROVIDER_TYPE = "provider"
@@ -2559,6 +2564,12 @@
 LOGWRITER_CEPHFS_WRITER = os.path.join(LOGWRITER_DIR, "cephfs.logwriter.yaml")
 LOGWRITER_STS_PATH = os.path.join(LOGWRITER_DIR, "logwriter.rbd.yaml")
 
+# Network Fence CRDs
+NETWORK_FENCE_CLASS_CRD = os.path.join(
+    TEMPLATE_CSI_ADDONS_DIR, "network-fence-class.yaml"
+)
+NETWORK_FENCE_CRD = os.path.join(TEMPLATE_CSI_ADDONS_DIR, "network-fence-rbd.yaml")
+
 # MCG namespace constants
 MCG_NS_AWS_ENDPOINT = "https://s3.amazonaws.com"
 MCG_NS_AZURE_ENDPOINT = "https://blob.core.windows.net"
@@ -3167,6 +3178,8 @@
     f"{ARBITER_ZONE}{DATA_ZONE_1}-{DATA_ZONE_1}{DATA_ZONE_2}"
 )
 
+NODE_OUT_OF_SERVICE_TAINT = "node.kubernetes.io/out-of-service=nodeshutdown:NoExecute"
+
 # Logwriter workload labels
 
 LOGWRITER_CEPHFS_LABEL = "app=logwriter-cephfs"
diff --git a/ocs_ci/ocs/resources/stretchcluster.py b/ocs_ci/ocs/resources/stretchcluster.py
index b6fa4ebaa98..534fa45a7ea 100644
--- a/ocs_ci/ocs/resources/stretchcluster.py
+++ b/ocs_ci/ocs/resources/stretchcluster.py
@@ -319,7 +319,7 @@ def get_logfile_map(self, label):
             self.logfile_map[label][0] = list(set(self.logfile_map[label][0]))
         logger.info(self.logfile_map[label][0])
 
-    @retry(UnexpectedBehaviour, tries=6, delay=5)
+    @retry(UnexpectedBehaviour, tries=8, delay=5)
     def get_logwriter_reader_pods(
         self,
         label,
diff --git a/ocs_ci/templates/CSI/addons/network-fence-class.yaml b/ocs_ci/templates/CSI/addons/network-fence-class.yaml
new file mode 100644
index 00000000000..eb0d44d145b
--- /dev/null
+++ b/ocs_ci/templates/CSI/addons/network-fence-class.yaml
@@ -0,0 +1,10 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
+kind: NetworkFenceClass
+metadata:
+  name: odf-networkfenceclass
+spec:
+  provisioner: openshift-storage.rbd.csi.ceph.com
+  parameters:
+    clusterID: openshift-storage
+    csiaddons.openshift.io/networkfence-secret-name: rook-csi-rbd-node
+    csiaddons.openshift.io/networkfence-secret-namespace: openshift-storage
diff --git a/ocs_ci/templates/CSI/addons/network-fence-rbd.yaml b/ocs_ci/templates/CSI/addons/network-fence-rbd.yaml
new file mode 100644
index 00000000000..dfbbcb7e45b
--- /dev/null
+++ b/ocs_ci/templates/CSI/addons/network-fence-rbd.yaml
@@ -0,0 +1,14 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
+kind: NetworkFence
+metadata:
+  name: <openshift-node-name>
+spec:
+  cidrs:
+  - <ip/32>
+  driver: openshift-storage.rbd.csi.ceph.com
+  fenceState: Fenced
+  parameters:
+    clusterID: openshift-storage
+  secret:
+    name: rook-csi-rbd-provisioner
+    namespace: openshift-storage
diff --git a/ocs_ci/templates/workloads/logwriter/cephfs.logreader.yaml b/ocs_ci/templates/workloads/logwriter/cephfs.logreader.yaml
index 2dacb6e5a30..537bb91b4a7 100644
--- a/ocs_ci/templates/workloads/logwriter/cephfs.logreader.yaml
+++ b/ocs_ci/templates/workloads/logwriter/cephfs.logreader.yaml
@@ -7,6 +7,7 @@ metadata:
 spec:
   completions: 6
   parallelism: 6
+  backoffLimit: 10
   completionMode: Indexed
   template:
     metadata:
diff --git a/tests/conftest.py b/tests/conftest.py
index 0e8d1eb90e7..c30c778a95d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -189,6 +189,7 @@
     modify_deployment_replica_count,
     modify_statefulset_replica_count,
     create_resource,
+    create_network_fence_class,
 )
 from ocs_ci.ocs.ceph_debug import CephObjectStoreTool, MonStoreTool, RookCephPlugin
 from ocs_ci.ocs.bucket_utils import get_rgw_restart_counts
@@ -8160,11 +8161,12 @@ def setup_logwriter_workload(request, teardown_factory):
 
     """
 
-    def factory(pvc, logwriter_path):
+    def factory(pvc, logwriter_path, zone_aware=True):
         """
         Args:
             pvc (PVC): PVC object
             logwriter_path (str): String representing logwriter yaml path
+            zone_aware (bool): True if workloads are zone aware else False
 
         Returns:
             OCS object: Lgwriter deployment object
@@ -8179,6 +8181,10 @@ def factory(pvc, logwriter_path):
         dc_data["spec"]["template"]["spec"]["volumes"][0]["persistentVolumeClaim"][
             "claimName"
         ] = pvc.name
+
+        if not zone_aware:
+            dc_data["spec"]["template"]["spec"].pop("topologySpreadConstraints")
+
         logwriter_dc = helpers.create_resource(**dc_data)
         teardown_factory(logwriter_dc)
 
@@ -8213,14 +8219,15 @@ def logreader_workload_factory(request, teardown_factory):
 
 
 def setup_logreader_workload(request, teardown_factory):
-    def factory(pvc, logreader_path, duration=30):
+    def factory(pvc, logreader_path, duration=30, zone_aware=True):
         """
         Args:
             pvc (PVC): PVC object
             logreader_path (str): String representing logreader yaml path
             duration (int): Time in minutes, representing read duration
+            zone_aware (bool): True if the workloads are zone aware False otherwise
 
-        Retuns:
+        Returns:
             OCS object: Logreader job object
 
         """
@@ -8238,6 +8245,10 @@ def factory(pvc, logreader_path, duration=30):
         job_data["spec"]["template"]["spec"]["containers"][0]["command"][
             2
         ] = f"/opt/logreader.py -t {duration} *.log -d"
+
+        if not zone_aware:
+            job_data["spec"]["template"]["spec"].pop("topologySpreadConstraints")
+
         logreader_job = helpers.create_resource(**job_data)
         teardown_factory(logreader_job)
 
@@ -8311,7 +8322,7 @@ def setup_logwriter_cephfs_workload(
 
     """
 
-    def factory(read_duration=30):
+    def factory(read_duration=30, **kwargs):
         """
         Args:
             read_duration (int): Time duration in minutes
@@ -8326,10 +8337,10 @@ def factory(read_duration=30):
             project_name=setup_stretch_cluster_project
         )
         logwriter_workload = logwriter_workload_factory(
-            pvc=pvc, logwriter_path=logwriter_path
+            pvc=pvc, logwriter_path=logwriter_path, **kwargs
         )
         logreader_workload = logreader_workload_factory(
-            pvc=pvc, logreader_path=logreader_path, duration=read_duration
+            pvc=pvc, logreader_path=logreader_path, duration=read_duration, **kwargs
         )
         return logwriter_workload, logreader_workload
 
@@ -8365,22 +8376,37 @@ def setup_logwriter_rbd_workload(
 
     """
 
-    logwriter_sts_path = constants.LOGWRITER_STS_PATH
-    sts_data = templating.load_yaml(logwriter_sts_path)
-    sts_data["metadata"]["namespace"] = setup_stretch_cluster_project.namespace
-    logwriter_sts = helpers.create_resource(**sts_data)
-    teardown_factory(logwriter_sts)
-    logwriter_sts_pods = [
-        pod["metadata"]["name"]
-        for pod in get_pods_having_label(
-            label="app=logwriter-rbd", namespace=setup_stretch_cluster_project.namespace
+    def factory(zone_aware=True):
+        """
+        Factory function to setup the logwriter rbd workloads
+
+        Args:
+            zone_aware (bool): True if the workloads are zone aware False otherwise
+
+        """
+        logwriter_sts_path = constants.LOGWRITER_STS_PATH
+        sts_data = templating.load_yaml(logwriter_sts_path)
+        sts_data["metadata"]["namespace"] = setup_stretch_cluster_project.namespace
+        if not zone_aware:
+            sts_data["spec"]["template"]["spec"].pop("topologySpreadConstraints")
+
+        logwriter_sts = helpers.create_resource(**sts_data)
+        teardown_factory(logwriter_sts)
+        logwriter_sts_pods = [
+            pod["metadata"]["name"]
+            for pod in get_pods_having_label(
+                label="app=logwriter-rbd",
+                namespace=setup_stretch_cluster_project.namespace,
+            )
+        ]
+        wait_for_pods_to_be_running(
+            namespace=setup_stretch_cluster_project.namespace,
+            pod_names=logwriter_sts_pods,
         )
-    ]
-    wait_for_pods_to_be_running(
-        namespace=setup_stretch_cluster_project.namespace, pod_names=logwriter_sts_pods
-    )
 
-    return logwriter_sts
+        return logwriter_sts
+
+    return factory
 
 
 @pytest.fixture(scope="session")
@@ -9715,3 +9741,38 @@ def admin_client():
     from ocp_resources.resource import get_client
 
     return get_client(config_file=kubeconfig_path)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
+
+    """
+    try:
+        network_fence_class = OCP(
+            kind=constants.NETWORK_FENCE_CLASS,
+            namespace=ocsci_config.ENV_DATA["cluster_namespace"],
+            resource_name=constants.ODF_NETWORK_FENCE_CLASS,
+        )
+        created_by_fixture = False
+        if not network_fence_class.get(dont_raise=True):
+            created_by_fixture = True
+            create_network_fence_class()
+        else:
+            log.info(
+                f"NetworkFenceClass {network_fence_class.resource_name} already exists!"
+            )
+    finally:
+
+        def finalizer():
+            """
+            Delete the NFC CRD if created by fixture
+
+            """
+            if created_by_fixture:
+                network_fence_class.delete(
+                    resource_name=constants.ODF_NETWORK_FENCE_CLASS
+                )
+
+        request.addfinalizer(finalizer)
diff --git a/tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py b/tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
new file mode 100644
index 00000000000..9d0a572ddb1
--- /dev/null
+++ b/tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
@@ -0,0 +1,361 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+    untaint_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+            """
+            Teardown function to remove the taints and unfence nodes
+
+            """
+            untaint_nodes(
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+                nodes_to_untaint=self.nodes_to_shutdown,
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+            for node_obj in self.nodes_to_shutdown:
+                unfence_node(node_obj.name, delete=True)
+            logger.info("Cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        self.nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=self.nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in self.nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in self.nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Check if all the pods are running
+        log.info(
+            "Checking if all the logwriter/logreader pods are relocated and successfully running"
+        )
+        sc_obj.get_logwriter_reader_pods(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logwriter_reader_pods(
+            label=constants.LOGREADER_CEPHFS_LABEL,
+            statuses=[constants.STATUS_RUNNING, constants.STATUS_COMPLETED],
+        )
+        try:
+            retry(UnexpectedBehaviour, tries=1)(sc_obj.get_logwriter_reader_pods)(
+                label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+            )
+        except UnexpectedBehaviour:
+            if not fencing:
+                log.info(
+                    "It is expected for RBD workload with RWO to stuck in terminating state"
+                )
+                log.info("Trying the workaround now...")
+                pods_terminating = [
+                    Pod(**pod_info)
+                    for pod_info in get_pods_having_label(
+                        label=constants.LOGWRITER_RBD_LABEL,
+                        namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+                    )
+                ]
+                log.info(pods_terminating)
+                for pod in pods_terminating:
+                    log.info(f"Force deleting the pod {pod.name}")
+                    pod.delete(force=True)
+                sc_obj.get_logwriter_reader_pods(
+                    label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+                )
+            else:
+                log.error(
+                    "Looks like pods are not running or not relocated even after fencing.. please check"
+                )
+                raise
+
+        if fencing:
+
+            # If fencing is True, then unfence the nodes once the pods are relocated
+            log.info(
+                "If fencing was done, then we need to unfence the nodes once the pods are relocated and running"
+            )
+            for node in self.nodes_to_shutdown:
+                if node.name not in worker_nodes:
+                    continue
+                unfence_node(node.name, delete=True)
+
+            # Remove the taints from the nodes that were shutdown
+            untaint_nodes(
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+                nodes_to_untaint=self.nodes_to_shutdown,
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+        # Start the nodes that were shutdown
+        log.info(f"Starting the {zone} nodes")
+        try:
+            nodes.start_nodes(nodes=self.nodes_to_shutdown)
+        except Exception:
+            log.error("Something went wrong while starting the nodes!")
+            raise
+
+        # Validate all nodes are in READY state and up
+        retry(
+            (
+                CommandFailed,
+                TimeoutError,
+                AssertionError,
+                ResourceWrongStatusException,
+            ),
+            tries=10,
+            delay=15,
+        )(wait_for_nodes_status(timeout=1800))
+        log.info(f"Nodes of zone {zone} are started successfully")
+
+        # Make sure all the logwriter pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("All logwriter workload pods are running!")
+
+        # check for any data loss through logwriter logs
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_CEPHFS_LABEL
+        ), "[CephFS] Data is lost"
+        log.info("[CephFS] No data loss is seen")
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_RBD_LABEL
+        ), "[RBD] Data is lost"
+        log.info("[RBD] No data loss is seen")
+
+        # check for data corruption through logreader logs
+        sc_obj.cephfs_logreader_job.delete()
+        for pod in sc_obj.cephfs_logreader_pods:
+            pod.wait_for_pod_delete(timeout=120)
+        log.info("All old CephFS logreader pods are deleted")
+        pvc = get_pvc_objs(
+            pvc_names=[
+                sc_obj.cephfs_logwriter_dep.get()["spec"]["template"]["spec"][
+                    "volumes"
+                ][0]["persistentVolumeClaim"]["claimName"]
+            ],
+            namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+        )[0]
+        logreader_workload_factory(
+            pvc=pvc, logreader_path=constants.LOGWRITER_CEPHFS_READER, duration=5
+        )
+        sc_obj.get_logwriter_reader_pods(constants.LOGREADER_CEPHFS_LABEL)
+
+        wait_for_pods_to_be_in_statuses(
+            expected_statuses=constants.STATUS_COMPLETED,
+            pod_names=[pod.name for pod in sc_obj.cephfs_logreader_pods],
+            timeout=900,
+            namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+        )
+        log.info("[CephFS] Logreader job pods have reached 'Completed' state!")
+
+        assert sc_obj.check_for_data_corruption(
+            label=constants.LOGREADER_CEPHFS_LABEL
+        ), "Data is corrupted for cephFS workloads"
+        log.info("No data corruption is seen in CephFS workloads")
+
+        assert sc_obj.check_for_data_corruption(
+            label=constants.LOGWRITER_RBD_LABEL
+        ), "Data is corrupted for RBD workloads"
+        log.info("No data corruption is seen in RBD workloads")


Reviews:
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-03T13:38:08Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-03T13:41:55Z
user: login: PrasadDesala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T15:02:25Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:02:51Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:05:00Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:05:17Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:08:13Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:08:55Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:22:11Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:27:18Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:47:56Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T05:49:08Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T10:47:54Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T10:53:15Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T10:56:03Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T11:03:00Z
user: login: avd-sagare
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T11:03:55Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T14:14:27Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T14:15:21Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T14:15:42Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T14:18:04Z
user: login: PrasadDesala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T07:07:04Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T11:14:07Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T11:15:31Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T06:00:41Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T06:02:05Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           mashetty-stm16
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/1d5144c) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T04:29:45Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           mashetty-stm16
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job PASSED](https://url.corp.redhat.com/1268d89).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T05:49:26Z
user: login: PrasadDesala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T07:37:24Z
user: login: PrasadDesala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T07:46:21Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T07:45:44Z
user: login: mashetty330
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T09:43:06Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           mashetty-stm22
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/b516ccb) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-22T04:31:52Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           mashetty-stm22
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/2e8b17b) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-22T14:04:44Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           mashetty-stm22
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job PASSED](https://url.corp.redhat.com/21e9b02).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-23T09:27:48Z
user: login: prsurve
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T06:24:33Z
user: login: PrasadDesala
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T15:17:35Z

Review comments:
diff_hunk: @@ -0,0 +1,346 @@
+import logging
+import pytest
+import time
+
+from datetime import datetime, timezone
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import taint_nodes, get_nodes, get_worker_nodes
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            master_nodes = get_nodes(node_type=constants.MASTER_MACHINE)
+            worker_nodes = get_nodes(node_type=constants.WORKER_MACHINE)
+            nodes_not_ready = list()
+            nodes_not_ready.extend(
+                [node for node in worker_nodes if node.status() != "Ready"]
+            )
+            nodes_not_ready.extend(
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: avd-sagare
body: combine these two list comprehensions into one ?
created_at: 2025-01-03T13:38:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,346 @@
+import logging
+import pytest
+import time
+
+from datetime import datetime, timezone
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import taint_nodes, get_nodes, get_worker_nodes
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            master_nodes = get_nodes(node_type=constants.MASTER_MACHINE)
+            worker_nodes = get_nodes(node_type=constants.WORKER_MACHINE)
+            nodes_not_ready = list()
+            nodes_not_ready.extend(
+                [node for node in worker_nodes if node.status() != "Ready"]
+            )
+            nodes_not_ready.extend(
+                [node for node in master_nodes if node.status() != "Ready"]
+            )
+
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=30,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(
+                True,
+            ),
+            pytest.param(
+                False,
+            ),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: avd-sagare
body: can you please add test steps here? It will be easy to check what this test does.
created_at: 2025-01-03T13:41:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,14 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
path: ocs_ci/templates/network/network-fence.yaml
user: login: PrasadDesala
body: is this something different from https://github.com/red-hat-storage/ocs-ci/blob/4c80d85935f9fea013d0cfa9c69859c039b2bb58/ocs_ci/templates/CSI/addons/NetworkFence.yaml? 
created_at: 2025-04-23T14:17:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7991,7 +7992,7 @@ def setup_logwriter_workload(request, teardown_factory):
 
     """
 
-    def factory(pvc, logwriter_path):
+    def factory(pvc, logwriter_path, zone_aware=True):
path: tests/conftest.py
user: login: PrasadDesala
body: update the factory doc string with the new arg details
created_at: 2025-04-23T14:19:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -8198,22 +8207,31 @@ def setup_logwriter_rbd_workload(
 
     """
 
-    logwriter_sts_path = constants.LOGWRITER_STS_PATH
-    sts_data = templating.load_yaml(logwriter_sts_path)
-    sts_data["metadata"]["namespace"] = setup_stretch_cluster_project.namespace
-    logwriter_sts = helpers.create_resource(**sts_data)
-    teardown_factory(logwriter_sts)
-    logwriter_sts_pods = [
-        pod["metadata"]["name"]
-        for pod in get_pods_having_label(
-            label="app=logwriter-rbd", namespace=setup_stretch_cluster_project.namespace
+    def factory(zone_aware=True):
+
path: tests/conftest.py
user: login: PrasadDesala
body: add doc strings
created_at: 2025-04-23T14:20:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
path: tests/conftest.py
user: login: PrasadDesala
body: Ideally, whatever resources were created as part of the test are expected to be destroyed after completing the test. Considering that, do we really need to handle the 'already exists' case?
created_at: 2025-04-23T14:30:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
+
+    """
+    try:
path: tests/conftest.py
user: login: PrasadDesala
body: this try finally can be removed if we ignore the  'already exists' case
created_at: 2025-04-23T14:32:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: can't this be replaced with restart_nodes_by_stop_and_start_teardown?
created_at: 2025-04-23T14:36:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: As discussed, this should be removed. Archiving the crashes can result in genuine bugs escaping. There is some work ongoing to handle this situation and bring the cluster health back to allow subsequent tests in the pipeline to run
created_at: 2025-04-23T14:40:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: add an empty line above
created_at: 2025-04-23T14:40:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: Should we first check if the node is fenced and if it is, then unfence it?
created_at: 2025-04-23T14:41:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: you are calling it but I don't see any usage of it in the test
created_at: 2025-04-23T14:48:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: You can remove these line if you use get_ocs_nodes_in_zone instead of get_nodes_in_zone 
created_at: 2025-04-23T14:53:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Wait for the buffer time of pod relocation
+        log.info("Wait until the pod relocation buffer time of 10 minutes")
+        time.sleep(600)
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: can this harcoded sleep be avoided? maybe through polling with some resources or using timeoutsampler ?
created_at: 2025-04-23T14:54:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: would unfence_node clear the added taints as well? If not then, we need to remove the added taints on those nodes in teardown
created_at: 2025-04-23T14:58:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
path: tests/conftest.py
user: login: mashetty330
body: @PrasadDesala Network fence class can be created and there is no need for it delete as it doesn't have any impact to the cluster or process unlike networkfence objects. so i just verify if it already exists then dont need to create it explicitly 
created_at: 2025-04-24T05:02:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: you cant stop already stopped nodes. it will fail teardown.
created_at: 2025-04-24T05:05:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: ack
created_at: 2025-04-24T05:05:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: if you are referring to `init_sanity` it is being used to check the sanity. since I don't have factory function I don't have to explicitly use the fixture in the code. it automatically gets executed
created_at: 2025-04-24T05:08:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: @PrasadDesala But I want to shutdown even the master nodes as well. 
created_at: 2025-04-24T05:08:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
+
+    """
+    try:
path: tests/conftest.py
user: login: mashetty330
body: try - finally block is not created for that. its created to make sure teardown code is registered even if there is a failure during network fence class setup.
created_at: 2025-04-24T05:22:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,14 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
path: ocs_ci/templates/network/network-fence.yaml
user: login: mashetty330
body: yes its different and looks very specific. using it would need lot of modification. 
created_at: 2025-04-24T05:27:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: @PrasadDesala will take care of it in this stretch cluster pr https://github.com/red-hat-storage/ocs-ci/pull/11392 as I will be moving these repetitive codes into common place
created_at: 2025-04-24T05:47:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,346 @@
+import logging
+import pytest
+import time
+
+from datetime import datetime, timezone
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import taint_nodes, get_nodes, get_worker_nodes
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            master_nodes = get_nodes(node_type=constants.MASTER_MACHINE)
+            worker_nodes = get_nodes(node_type=constants.WORKER_MACHINE)
+            nodes_not_ready = list()
+            nodes_not_ready.extend(
+                [node for node in worker_nodes if node.status() != "Ready"]
+            )
+            nodes_not_ready.extend(
+                [node for node in master_nodes if node.status() != "Ready"]
+            )
+
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=30,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(
+                True,
+            ),
+            pytest.param(
+                False,
+            ),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: Done
created_at: 2025-04-24T05:49:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6001,3 +6002,118 @@ def find_cephfilesystemsubvolumegroup(storageclient_uid=None):
         cephbfssubvolumegroup = storage_consumer.get_cephfs_subvolumegroup()
 
     return cephbfssubvolumegroup
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-csi-rbdplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
path: ocs_ci/helpers/helpers.py
user: login: avd-sagare
body: I guess it shoud be "network_fence_obj" instead of OCS
created_at: 2025-04-24T10:47:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6001,3 +6002,118 @@ def find_cephfilesystemsubvolumegroup(storageclient_uid=None):
         cephbfssubvolumegroup = storage_consumer.get_cephfs_subvolumegroup()
 
     return cephbfssubvolumegroup
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-csi-rbdplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
+
+    """
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE,
+        namespace=config.ENV_DATA["cluster_namespace"],
+        resource_name=node_name,
+    )
+
+    if not network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        logger.info("Creating NetworkFence")
+        network_fence_dict = templating.load_yaml(constants.NETWORK_FENCE_CRD)
+        network_fence_dict["metadata"]["name"] = node_name
+        network_fence_dict["spec"]["cidrs"][0] = cidr
+        network_fence_obj = create_resource(**network_fence_dict)
+        if network_fence_obj.ocp.get(
+            resource_name=network_fence_obj.name, dont_raise=True
+        ):
+            logger.info(
+                f"NetworkFence {network_fence_obj.name} for node {node_name} created successfully"
+            )
+    else:
+        logger.info(f"Network fence object for {node_name} already exists!")
+    return network_fence_obj
+
+
+def unfence_node(node_name, delete=False):
+    """
+    Un-fence node
+
+    Args:
+        node_name (str): Name of the node
+        delete (bool): If True, delete the network fence object
+
+    """
+
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE, namespace=config.ENV_DATA["cluster_namespace"]
+    )
+
+    if network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        network_fence_obj.patch(
+            resource_name=node_name,
+            params='{"spec":{"fenceState":"Unfenced"}}',
+            format_type="merge",
+        )
+        assert (
+            network_fence_obj.get(resource_name=node_name)["spec"]["fenceState"]
+            != "Fenced"
+        ), f"{node_name} doesnt seem to be unfenced"
path: ocs_ci/helpers/helpers.py
user: login: avd-sagare
body: can we make this message more  clear .like expected to be unfenced but its still fenced
created_at: 2025-04-24T10:53:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,14 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
+kind: NetworkFence
+metadata:
+  name: <openshift-node-name>
+spec:
+  cidrs:
+  - <ip/32>
+  driver: openshift-storage.rbd.csi.ceph.com
+  fenceState: Fenced
path: ocs_ci/templates/CSI/addons/network-fence-rbd.yaml
user: login: avd-sagare
body: is this by default value ?or we are suppose to add  while executing tc?
I am just relating this with DR. we add this line while doing fencing.
created_at: 2025-04-24T10:56:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,358 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+
+            fenced_nodes = [node.name for node in self.nodes_to_shutdown]
+            taint_nodes(
+                nodes=fenced_nodes,
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+            for node_name in fenced_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("Cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        self.nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=self.nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in self.nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in self.nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Check if all the pods are running
+        log.info(
+            "Checking if all the logwriter/logreader pods are relocated and successfully running"
+        )
+        sc_obj.get_logwriter_reader_pods(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logwriter_reader_pods(
+            label=constants.LOGREADER_CEPHFS_LABEL,
+            statuses=[constants.STATUS_RUNNING, constants.STATUS_COMPLETED],
+        )
+        try:
+            retry(UnexpectedBehaviour, tries=1)(sc_obj.get_logwriter_reader_pods)(
+                label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+            )
+        except UnexpectedBehaviour:
+            if not fencing:
+                log.info(
+                    "It is expected for RBD workload with RWO to stuck in terminating state"
+                )
+                log.info("Trying the workaround now...")
+                pods_terminating = [
+                    Pod(**pod_info)
+                    for pod_info in get_pods_having_label(
+                        label=constants.LOGWRITER_RBD_LABEL,
+                        namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+                    )
+                ]
+                log.info(pods_terminating)
+                for pod in pods_terminating:
+                    log.info(f"Force deleting the pod {pod.name}")
+                    pod.delete(force=True)
+                sc_obj.get_logwriter_reader_pods(
+                    label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+                )
+            else:
+                log.error(
+                    "Looks like pods are not running or not relocated even after fencing.. please check"
+                )
+                raise
+
+        if fencing:
+
+            # If fencing is True, then unfence the nodes once the pods are relocated
+            log.info(
+                "If fencing was done, then we need to unfence the nodes once the pods are relocated and running"
+            )
+            for node in self.nodes_to_shutdown:
+                if node.name not in worker_nodes:
+                    continue
+                unfence_node(node.name, delete=True)
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: avd-sagare
body: can we check fencing was done on node or not before trying for unfence the nodes
created_at: 2025-04-24T11:03:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,358 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+
+            fenced_nodes = [node.name for node in self.nodes_to_shutdown]
+            taint_nodes(
+                nodes=fenced_nodes,
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+            for node_name in fenced_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("Cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        self.nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=self.nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in self.nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in self.nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Check if all the pods are running
+        log.info(
+            "Checking if all the logwriter/logreader pods are relocated and successfully running"
+        )
+        sc_obj.get_logwriter_reader_pods(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logwriter_reader_pods(
+            label=constants.LOGREADER_CEPHFS_LABEL,
+            statuses=[constants.STATUS_RUNNING, constants.STATUS_COMPLETED],
+        )
+        try:
+            retry(UnexpectedBehaviour, tries=1)(sc_obj.get_logwriter_reader_pods)(
+                label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+            )
+        except UnexpectedBehaviour:
+            if not fencing:
+                log.info(
+                    "It is expected for RBD workload with RWO to stuck in terminating state"
+                )
+                log.info("Trying the workaround now...")
+                pods_terminating = [
+                    Pod(**pod_info)
+                    for pod_info in get_pods_having_label(
+                        label=constants.LOGWRITER_RBD_LABEL,
+                        namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+                    )
+                ]
+                log.info(pods_terminating)
+                for pod in pods_terminating:
+                    log.info(f"Force deleting the pod {pod.name}")
+                    pod.delete(force=True)
+                sc_obj.get_logwriter_reader_pods(
+                    label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+                )
+            else:
+                log.error(
+                    "Looks like pods are not running or not relocated even after fencing.. please check"
+                )
+                raise
+
+        if fencing:
+
+            # If fencing is True, then unfence the nodes once the pods are relocated
+            log.info(
+                "If fencing was done, then we need to unfence the nodes once the pods are relocated and running"
+            )
+            for node in self.nodes_to_shutdown:
+                if node.name not in worker_nodes:
+                    continue
+                unfence_node(node.name, delete=True)
+
+            # Remove the taints from the nodes that were shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+        # Start the nodes that were shutdown
+        log.info(f"Starting the {zone} nodes")
+        try:
+            nodes.start_nodes(nodes=self.nodes_to_shutdown)
+        except Exception:
+            log.error("Something went wrong while starting the nodes!")
+            raise
+
+        # Validate all nodes are in READY state and up
+        retry(
+            (
+                CommandFailed,
+                TimeoutError,
+                AssertionError,
+                ResourceWrongStatusException,
+            ),
+            tries=10,
+            delay=15,
+        )(wait_for_nodes_status(timeout=1800))
+        log.info(f"Nodes of zone {zone} are started successfully")
+
+        # Make sure all the logwriter pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("All logwriter workload pods are running!")
+
+        # check for any data loss through logwriter logs
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_CEPHFS_LABEL
+        ), "[CephFS] Data is lost"
+        log.info("[CephFS] No data loss is seen")
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_RBD_LABEL
+        ), "[RBD] Data is lost"
+        log.info("[RBD] No data loss is seen")
+
+        # check for data corruption through logreader logs
+        sc_obj.cephfs_logreader_job.delete()
+        for pod in sc_obj.cephfs_logreader_pods:
+            pod.wait_for_pod_delete(timeout=120)
+        log.info("All old CephFS logreader pods are deleted")
+        pvc = get_pvc_objs(
+            pvc_names=[
+                sc_obj.cephfs_logwriter_dep.get()["spec"]["template"]["spec"][
+                    "volumes"
+                ][0]["persistentVolumeClaim"]["claimName"]
+            ],
+            namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+        )[0]
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: avd-sagare
body: can we avoid indexing ?

created_at: 2025-04-24T11:03:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6001,3 +6002,118 @@ def find_cephfilesystemsubvolumegroup(storageclient_uid=None):
         cephbfssubvolumegroup = storage_consumer.get_cephfs_subvolumegroup()
 
     return cephbfssubvolumegroup
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-csi-rbdplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
path: ocs_ci/helpers/helpers.py
user: login: mashetty330
body: @avd-sagare No its OCS object of kind NetworkFence
created_at: 2025-04-24T14:14:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,14 @@
+apiVersion: csiaddons.openshift.io/v1alpha1
+kind: NetworkFence
+metadata:
+  name: <openshift-node-name>
+spec:
+  cidrs:
+  - <ip/32>
+  driver: openshift-storage.rbd.csi.ceph.com
+  fenceState: Fenced
path: ocs_ci/templates/CSI/addons/network-fence-rbd.yaml
user: login: mashetty330
body: No this is just default value. As the first time we create it is meant to be fenced.
created_at: 2025-04-24T14:15:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,358 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+
+            fenced_nodes = [node.name for node in self.nodes_to_shutdown]
+            taint_nodes(
+                nodes=fenced_nodes,
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+            for node_name in fenced_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("Cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        self.nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=self.nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in self.nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in self.nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Check if all the pods are running
+        log.info(
+            "Checking if all the logwriter/logreader pods are relocated and successfully running"
+        )
+        sc_obj.get_logwriter_reader_pods(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logwriter_reader_pods(
+            label=constants.LOGREADER_CEPHFS_LABEL,
+            statuses=[constants.STATUS_RUNNING, constants.STATUS_COMPLETED],
+        )
+        try:
+            retry(UnexpectedBehaviour, tries=1)(sc_obj.get_logwriter_reader_pods)(
+                label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+            )
+        except UnexpectedBehaviour:
+            if not fencing:
+                log.info(
+                    "It is expected for RBD workload with RWO to stuck in terminating state"
+                )
+                log.info("Trying the workaround now...")
+                pods_terminating = [
+                    Pod(**pod_info)
+                    for pod_info in get_pods_having_label(
+                        label=constants.LOGWRITER_RBD_LABEL,
+                        namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+                    )
+                ]
+                log.info(pods_terminating)
+                for pod in pods_terminating:
+                    log.info(f"Force deleting the pod {pod.name}")
+                    pod.delete(force=True)
+                sc_obj.get_logwriter_reader_pods(
+                    label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+                )
+            else:
+                log.error(
+                    "Looks like pods are not running or not relocated even after fencing.. please check"
+                )
+                raise
+
+        if fencing:
+
+            # If fencing is True, then unfence the nodes once the pods are relocated
+            log.info(
+                "If fencing was done, then we need to unfence the nodes once the pods are relocated and running"
+            )
+            for node in self.nodes_to_shutdown:
+                if node.name not in worker_nodes:
+                    continue
+                unfence_node(node.name, delete=True)
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: its been taken care inside unfence_node()
created_at: 2025-04-24T14:15:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,358 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+
+            fenced_nodes = [node.name for node in self.nodes_to_shutdown]
+            taint_nodes(
+                nodes=fenced_nodes,
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+            for node_name in fenced_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("Cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        self.nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=self.nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in self.nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in self.nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
+
+                # Fetch the cidrs for creating network fence
+                cidrs = retry(CommandFailed, tries=5)(
+                    get_rbd_daemonset_csi_addons_node_object
+                )(node.name)["status"]["networkFenceClientStatus"][0]["ClientDetails"][
+                    0
+                ][
+                    "cidrs"
+                ]
+
+                # Create the network fence
+                retry(CommandFailed, tries=5)(create_network_fence)(
+                    node.name, cidr=cidrs[0]
+                )
+
+            # Taint the nodes that are shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=constants.NODE_OUT_OF_SERVICE_TAINT,
+            )
+
+        # Check if all the pods are running
+        log.info(
+            "Checking if all the logwriter/logreader pods are relocated and successfully running"
+        )
+        sc_obj.get_logwriter_reader_pods(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logwriter_reader_pods(
+            label=constants.LOGREADER_CEPHFS_LABEL,
+            statuses=[constants.STATUS_RUNNING, constants.STATUS_COMPLETED],
+        )
+        try:
+            retry(UnexpectedBehaviour, tries=1)(sc_obj.get_logwriter_reader_pods)(
+                label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+            )
+        except UnexpectedBehaviour:
+            if not fencing:
+                log.info(
+                    "It is expected for RBD workload with RWO to stuck in terminating state"
+                )
+                log.info("Trying the workaround now...")
+                pods_terminating = [
+                    Pod(**pod_info)
+                    for pod_info in get_pods_having_label(
+                        label=constants.LOGWRITER_RBD_LABEL,
+                        namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+                    )
+                ]
+                log.info(pods_terminating)
+                for pod in pods_terminating:
+                    log.info(f"Force deleting the pod {pod.name}")
+                    pod.delete(force=True)
+                sc_obj.get_logwriter_reader_pods(
+                    label=constants.LOGWRITER_RBD_LABEL, exp_num_replicas=2
+                )
+            else:
+                log.error(
+                    "Looks like pods are not running or not relocated even after fencing.. please check"
+                )
+                raise
+
+        if fencing:
+
+            # If fencing is True, then unfence the nodes once the pods are relocated
+            log.info(
+                "If fencing was done, then we need to unfence the nodes once the pods are relocated and running"
+            )
+            for node in self.nodes_to_shutdown:
+                if node.name not in worker_nodes:
+                    continue
+                unfence_node(node.name, delete=True)
+
+            # Remove the taints from the nodes that were shutdown
+            taint_nodes(
+                nodes=[node.name for node in self.nodes_to_shutdown],
+                taint_label=f"{constants.NODE_OUT_OF_SERVICE_TAINT}-",
+            )
+            log.info("Successfully removed taints from the nodes that were shutdown")
+
+        # Start the nodes that were shutdown
+        log.info(f"Starting the {zone} nodes")
+        try:
+            nodes.start_nodes(nodes=self.nodes_to_shutdown)
+        except Exception:
+            log.error("Something went wrong while starting the nodes!")
+            raise
+
+        # Validate all nodes are in READY state and up
+        retry(
+            (
+                CommandFailed,
+                TimeoutError,
+                AssertionError,
+                ResourceWrongStatusException,
+            ),
+            tries=10,
+            delay=15,
+        )(wait_for_nodes_status(timeout=1800))
+        log.info(f"Nodes of zone {zone} are started successfully")
+
+        # Make sure all the logwriter pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("All logwriter workload pods are running!")
+
+        # check for any data loss through logwriter logs
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_CEPHFS_LABEL
+        ), "[CephFS] Data is lost"
+        log.info("[CephFS] No data loss is seen")
+        assert sc_obj.check_for_data_loss(
+            constants.LOGWRITER_RBD_LABEL
+        ), "[RBD] Data is lost"
+        log.info("[RBD] No data loss is seen")
+
+        # check for data corruption through logreader logs
+        sc_obj.cephfs_logreader_job.delete()
+        for pod in sc_obj.cephfs_logreader_pods:
+            pod.wait_for_pod_delete(timeout=120)
+        log.info("All old CephFS logreader pods are deleted")
+        pvc = get_pvc_objs(
+            pvc_names=[
+                sc_obj.cephfs_logwriter_dep.get()["spec"]["template"]["spec"][
+                    "volumes"
+                ][0]["persistentVolumeClaim"]["claimName"]
+            ],
+            namespace=constants.STRETCH_CLUSTER_NAMESPACE,
+        )[0]
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: why? i didn't understand the problem with indexing. 
created_at: 2025-04-24T14:18:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7991,11 +7992,12 @@ def setup_logwriter_workload(request, teardown_factory):
 
     """
 
-    def factory(pvc, logwriter_path):
+    def factory(pvc, logwriter_path, zone_aware=True):
         """
         Args:
             pvc (PVC): PVC object
             logwriter_path (str): String representing logwriter yaml path
+            zone_aware (boolean): True if workloads are zone aware else False
path: tests/conftest.py
user: login: PrasadDesala
body: ```suggestion
            zone_aware (bool): True if workloads are zone aware else False
```
created_at: 2025-04-25T06:30:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -8198,22 +8208,37 @@ def setup_logwriter_rbd_workload(
 
     """
 
-    logwriter_sts_path = constants.LOGWRITER_STS_PATH
-    sts_data = templating.load_yaml(logwriter_sts_path)
-    sts_data["metadata"]["namespace"] = setup_stretch_cluster_project.namespace
-    logwriter_sts = helpers.create_resource(**sts_data)
-    teardown_factory(logwriter_sts)
-    logwriter_sts_pods = [
-        pod["metadata"]["name"]
-        for pod in get_pods_having_label(
-            label="app=logwriter-rbd", namespace=setup_stretch_cluster_project.namespace
+    def factory(zone_aware=True):
+        """
+        Factory function to setup the logwriter rbd workloads
+
+        Args:
+            zone_aware (boolean): True if the workloads are zone aware False otherwise
path: tests/conftest.py
user: login: PrasadDesala
body: ```suggestion
            zone_aware (bool): True if the workloads are zone aware False otherwise
```
created_at: 2025-04-25T06:30:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
path: tests/conftest.py
user: login: PrasadDesala
body: Though the resource has no impact on the cluster, as I mentioned above, it is best practice to delete all resources created during the tests after completion. This is not a blocker comment and can be revisited later
created_at: 2025-04-25T06:37:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: I did not understand, can you elaborate?
created_at: 2025-04-25T07:00:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,358 @@
+import logging
+import pytest
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    nodes_to_shutdown = []
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def remove_taint_unfence_teardown(self, request):
+        """
+        In case of failure in between test run, unfence the networkfence,
+        delete the NetworkFence objects and remove the taints
+
+        """
+
+        def teardown():
+
+            fenced_nodes = [node.name for node in self.nodes_to_shutdown]
+            taint_nodes(
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: you can use untaint_nodes function
created_at: 2025-04-25T07:01:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: what sanity are you doing? It appears like you are just initializing the sanity class
created_at: 2025-04-25T07:04:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: line 208, you are ignoring the master nodes. can you point me the code where you are doing master nodes shutdown
created_at: 2025-04-25T07:05:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9483,3 +9501,38 @@ def teardown():
                 nfs_sc_obj.delete()
 
         request.addfinalizer(teardown)
+
+
+@pytest.fixture(scope="session")
+def setup_network_fence_class(request):
+    """
+    Setup NetworkFenceClass CRD for ODF if not present
path: tests/conftest.py
user: login: mashetty330
body: @PrasadDesala here i'm only making sure that fixture creates network-fence-class only if it is not present. Also delete it if it is created as part of fixture.
created_at: 2025-04-29T11:14:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: Here im making sure if the nodes are shutdown then start them. this can be caused if the test fails in between the test after the nodes are down. so if i try to stop the node which is already shutdown it will throw an exception and fail the whole teardown.
created_at: 2025-04-29T11:15:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: yeah correct but we use it to check the health during the teardown.
created_at: 2025-04-30T06:00:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
+                    delay=15,
+                )(wait_for_nodes_status(timeout=1800))
+                log.info(
+                    f"Following nodes {nodes_not_ready} were NOT READY, are now in READY state"
+                )
+            else:
+                log.info("All nodes are READY")
+
+            # check cluster health
+            try:
+                log.info("Making sure ceph health is OK")
+                self.sanity_helpers.health_check(tries=50, cluster_check=False)
+            except CephHealthException as e:
+                assert (
+                    "HEALTH_WARN" in e.args[0]
+                ), f"Ignoring Ceph health warnings: {e.args[0]}"
+                get_ceph_tools_pod().exec_ceph_cmd(ceph_cmd="ceph crash archive-all")
+                log.info("Archived ceph crash!")
+
+        request.addfinalizer(finalizer)
+
+    @pytest.fixture(autouse=True)
+    def unfence_teardown(self, request):
+        """
+        In case of failure in between test run unfence the networkfence
+        and delete the NetworkFence objects
+
+        """
+
+        def teardown():
+            all_worker_nodes = get_worker_nodes()
+            for node_name in all_worker_nodes:
+                unfence_node(node_name, delete=True)
+            logger.info("cleaned up all network fence objects if any")
+
+        request.addfinalizer(teardown)
+
+    @pytest.mark.parametrize(
+        argnames="fencing",
+        argvalues=[
+            pytest.param(True, marks=[pytest.mark.polarion_id("OCS-6798")]),
+            pytest.param(False, marks=[pytest.mark.polarion_id("OCS-6799")]),
+        ],
+        ids=[
+            "With-Fencing",
+            "Without-Fencing",
+        ],
+    )
+    def test_zone_shutdowns(
+        self,
+        init_sanity,
+        setup_logwriter_cephfs_workload_factory,
+        setup_logwriter_rbd_workload_factory,
+        logreader_workload_factory,
+        setup_network_fence_class,
+        nodes,
+        fencing,
+    ):
+        """
+        This tests the zone unware app pod failover when there is
+        zone shutdown under the following scenarios:
+            1. When network fencing is done
+            2. When network fencing is not done
+
+        """
+
+        sc_obj = StretchCluster()
+
+        # Deploy the zone un-aware logwriter workloads
+        (
+            sc_obj.cephfs_logwriter_dep,
+            sc_obj.cephfs_logreader_job,
+        ) = setup_logwriter_cephfs_workload_factory(read_duration=0, zone_aware=False)
+
+        sc_obj.rbd_logwriter_sts = setup_logwriter_rbd_workload_factory(
+            zone_aware=False
+        )
+
+        # Fetch all the worker node names and zone to
+        # shutdown randomly
+        worker_nodes = get_worker_nodes()
+        zone = random.choice(constants.DATA_ZONE_LABELS)
+
+        # Make sure logwriter workload pods are running
+        check_for_logwriter_workload_pods(sc_obj, nodes=nodes)
+        log.info("Both logwriter CephFS and RBD workloads are in healthy state")
+
+        # Fetch logfile details to verify data integrity post recovery
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_CEPHFS_LABEL)
+        sc_obj.get_logfile_map(label=constants.LOGWRITER_RBD_LABEL)
+        log.info(
+            "Fetched the logfile details for data integrity verification post recovery"
+        )
+
+        # Shutdown the nodes
+        nodes_to_shutdown = sc_obj.get_nodes_in_zone(zone)
+        nodes.stop_nodes(nodes=nodes_to_shutdown)
+        wait_for_nodes_status(
+            node_names=[node.name for node in nodes_to_shutdown],
+            status=constants.NODE_NOT_READY,
+            timeout=300,
+        )
+        log.info(f"Nodes of zone {zone} are shutdown successfully")
+
+        if fencing:
+
+            # If fencing is True, then we need to fence the nodes after shutdown
+            log.info(
+                "Since fencing is enabled, we need to fence the nodes after zone shutdown"
+            )
+            for node in nodes_to_shutdown:
+
+                # Ignore the master nodes
+                if node.name not in worker_nodes:
+                    continue
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: At 191, im fetching all the nodes in a zone that is including master node. 
At 192, Im shutting down all the nodes in that zone including master node.
created_at: 2025-04-30T06:02:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: PrasadDesala
body: restart_nodes_by_stop_and_start_teardown() checks for the stopped VMs and start those making sure that all the VMs are up by the end of the test. It won't stop the nodes again which are already shutdown. Please check it again.
created_at: 2025-05-16T07:37:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6001,3 +6002,118 @@ def find_cephfilesystemsubvolumegroup(storageclient_uid=None):
         cephbfssubvolumegroup = storage_consumer.get_cephfs_subvolumegroup()
 
     return cephbfssubvolumegroup
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-openshift-storage.rbd.csi.ceph.com-nodeplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
+
+    """
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE,
+        namespace=config.ENV_DATA["cluster_namespace"],
+        resource_name=node_name,
+    )
+
+    if not network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        logger.info("Creating NetworkFence")
+        network_fence_dict = templating.load_yaml(constants.NETWORK_FENCE_CRD)
+        network_fence_dict["metadata"]["name"] = node_name
+        network_fence_dict["spec"]["cidrs"][0] = cidr
+        network_fence_obj = create_resource(**network_fence_dict)
+        if network_fence_obj.ocp.get(
+            resource_name=network_fence_obj.name, dont_raise=True
+        ):
+            logger.info(
+                f"NetworkFence {network_fence_obj.name} for node {node_name} created successfully"
+            )
+    else:
+        logger.info(f"Network fence object for {node_name} already exists!")
+    return network_fence_obj
+
+
+def unfence_node(node_name, delete=False):
+    """
+    Un-fence node
+
+    Args:
+        node_name (str): Name of the node
+        delete (bool): If True, delete the network fence object
+
+    """
+
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE, namespace=config.ENV_DATA["cluster_namespace"]
+    )
+
+    if network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        network_fence_obj.patch(
+            resource_name=node_name,
+            params='{"spec":{"fenceState":"Unfenced"}}',
+            format_type="merge",
+        )
+        assert (
+            network_fence_obj.get(resource_name=node_name)["spec"]["fenceState"]
+            != "Fenced"
path: ocs_ci/helpers/helpers.py
user: login: PrasadDesala
body: please use constant: ACTION_FENCE
created_at: 2025-05-16T07:42:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6001,3 +6002,118 @@ def find_cephfilesystemsubvolumegroup(storageclient_uid=None):
         cephbfssubvolumegroup = storage_consumer.get_cephfs_subvolumegroup()
 
     return cephbfssubvolumegroup
+
+
+def get_rbd_daemonset_csi_addons_node_object(node):
+    """
+    Gets rdb daemonset CSI addons node data
+
+    Args:
+        node (str): Name of the node
+
+    Returns:
+       dict: CSI addons node object info
+
+    """
+    namespace = config.ENV_DATA["cluster_namespace"]
+    csi_addons_node = OCP(kind=constants.CSI_ADDONS_NODE_KIND, namespace=namespace)
+    csi_addons_node_data = csi_addons_node.get(
+        resource_name=f"{node}-{namespace}-daemonset-openshift-storage.rbd.csi.ceph.com-nodeplugin"
+    )
+    return csi_addons_node_data
+
+
+def create_network_fence_class():
+    """
+    Create NetworkFenceClass CR and verify Ips are populated
+    in respective CsiAddonsNode objects
+
+    """
+
+    logger.info("Creating NetworkFenceClass")
+    network_fence_class_dict = templating.load_yaml(constants.NETWORK_FENCE_CLASS_CRD)
+    network_fence_class_obj = create_resource(**network_fence_class_dict)
+    if network_fence_class_obj.ocp.get(
+        resource_name=network_fence_class_obj.name, dont_raise=True
+    ):
+        logger.info(
+            f"NetworkFenceClass {network_fence_class_obj.name} created successfully"
+        )
+
+    logger.info("Verifying CsiAddonsNode object for CSI RBD daemonset")
+    all_nodes = get_worker_nodes()
+
+    for node_name in all_nodes:
+        cidrs = get_rbd_daemonset_csi_addons_node_object(node_name)["status"][
+            "networkFenceClientStatus"
+        ][0]["ClientDetails"][0]["cidrs"]
+        assert len(cidrs) == 1, "No cidrs are populated to CSI Addons node object"
+        logger.info(f"Cidr: {cidrs[0]} populated in {node_name} CSI addons node object")
+
+
+def create_network_fence(node_name, cidr):
+    """
+    Create NetworkFence for the node
+
+    Args:
+        node_name (str): Name of the node
+        cidr (str): cidr
+
+    Returns:
+        OCS: NetworkFence object
+
+    """
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE,
+        namespace=config.ENV_DATA["cluster_namespace"],
+        resource_name=node_name,
+    )
+
+    if not network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        logger.info("Creating NetworkFence")
+        network_fence_dict = templating.load_yaml(constants.NETWORK_FENCE_CRD)
+        network_fence_dict["metadata"]["name"] = node_name
+        network_fence_dict["spec"]["cidrs"][0] = cidr
+        network_fence_obj = create_resource(**network_fence_dict)
+        if network_fence_obj.ocp.get(
+            resource_name=network_fence_obj.name, dont_raise=True
+        ):
+            logger.info(
+                f"NetworkFence {network_fence_obj.name} for node {node_name} created successfully"
+            )
+    else:
+        logger.info(f"Network fence object for {node_name} already exists!")
+    return network_fence_obj
+
+
+def unfence_node(node_name, delete=False):
+    """
+    Un-fence node
+
+    Args:
+        node_name (str): Name of the node
+        delete (bool): If True, delete the network fence object
+
+    """
+
+    network_fence_obj = OCP(
+        kind=constants.NETWORK_FENCE, namespace=config.ENV_DATA["cluster_namespace"]
+    )
+
+    if network_fence_obj.get(resource_name=node_name, dont_raise=True):
+        network_fence_obj.patch(
+            resource_name=node_name,
+            params='{"spec":{"fenceState":"Unfenced"}}',
path: ocs_ci/helpers/helpers.py
user: login: PrasadDesala
body: please use constant ACTION_UNFENCE
created_at: 2025-05-16T07:42:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: ack. I was referring to `restarts_nodes_by_stop_and_start`
created_at: 2025-05-20T07:45:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,354 @@
+import logging
+import pytest
+import time
+import random
+
+from ocs_ci.framework.pytest_customization.marks import (
+    stretchcluster_required,
+    tier1,
+    turquoise_squad,
+)
+from ocs_ci.helpers.sanity_helpers import Sanity
+from ocs_ci.ocs.node import (
+    taint_nodes,
+    get_worker_nodes,
+    get_node_objs,
+    get_all_nodes,
+)
+from ocs_ci.helpers.helpers import (
+    create_network_fence,
+    get_rbd_daemonset_csi_addons_node_object,
+    unfence_node,
+)
+from ocs_ci.helpers.stretchcluster_helper import check_for_logwriter_workload_pods
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.exceptions import (
+    UnexpectedBehaviour,
+    CommandFailed,
+    ResourceWrongStatusException,
+    CephHealthException,
+)
+from ocs_ci.ocs.node import wait_for_nodes_status
+from ocs_ci.ocs.resources.pod import (
+    get_pods_having_label,
+    Pod,
+    get_ceph_tools_pod,
+    wait_for_pods_to_be_in_statuses,
+    logger,
+)
+from ocs_ci.ocs.resources.pvc import get_pvc_objs
+from ocs_ci.ocs.resources.stretchcluster import StretchCluster
+from ocs_ci.utility.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+@tier1
+@stretchcluster_required
+@turquoise_squad
+class TestZoneUnawareApps:
+
+    @pytest.fixture()
+    def init_sanity(self, request, nodes):
+        """
+        Initial Cluster sanity
+        """
+        self.sanity_helpers = Sanity()
+
+        def finalizer():
+            """
+            Make sure all the nodes are Running and
+            the ceph health is OK at the end of the test
+            """
+
+            # check if all the nodes are Running
+            log.info("Checking if all the nodes are READY")
+            nodes_not_ready = [
+                node_obj
+                for node_obj in get_node_objs(get_all_nodes())
+                if node_obj.status() != constants.STATUS_READY
+            ]
+            if len(nodes_not_ready) != 0:
+                try:
+                    nodes.start_nodes(nodes=nodes_not_ready)
+                except Exception:
+                    log.error(
+                        f"Something went wrong while starting the nodes {nodes_not_ready}!"
+                    )
+                    raise
+
+                retry(
+                    (
+                        CommandFailed,
+                        TimeoutError,
+                        AssertionError,
+                        ResourceWrongStatusException,
+                    ),
+                    tries=10,
path: tests/functional/disaster-recovery/sc_arbiter/test_zone_unaware_app.py
user: login: mashetty330
body: @PrasadDesala will take care of it in this stretch cluster pr https://github.com/red-hat-storage/ocs-ci/pull/11392 as I will be moving these repetitive codes into common place
created_at: 2025-05-20T09:43:06Z
author_association: CONTRIBUTOR

PR comments:
user: login: openshift-ci[bot]
created_at: 2025-05-27T15:17:41Z
body: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11000#" title="Author self-approved">mashetty330</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11000#pullrequestreview-2871527737" title="Approved">PrasadDesala</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11000#pullrequestreview-2869748009" title="Approved">prsurve</a>*


The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=red-hat-storage%2Focs-ci).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/red-hat-storage/ocs-ci/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE