Repository: red-hat-storage/ocs-ci
Pull request #11925: Resiliency test automation when platform under stress
Author: login: paraggit
Labels: name: Verified
description: Mark when PR was verified and log provided
name: size/XL
name: lgtm
Created: 2025-04-15T16:13:32Z | Merged: 2025-04-23T09:32:40Z


This PR introduces the resiliency scenario automation under platform stress . following stress conditions are induce to node-level using stress-ng tool.

Stress types:
- CPU stress: stress-ng --cpu 4 --cpu-load 70 --timeout 60s
- Memory stress: stress-ng --vm 4 --vm-bytes 70% --timeout 60s
- IO stress: stress-ng --hdd 4 --hdd-bytes 2G --timeout 60s
- Network stress: stress-ng --sock 4 --timeout 60s
- All combined stress: stress-ng --cpu 1 --vm 1 --io 1 --sock 1 --timeout 60s



Diff:
diff --git a/ocs_ci/ocs/ocp.py b/ocs_ci/ocs/ocp.py
index 6396785c3d3..8bc4e40ae57 100644
--- a/ocs_ci/ocs/ocp.py
+++ b/ocs_ci/ocs/ocp.py
@@ -238,7 +238,9 @@ def exec_oc_cmd(
             config.switch_ctx(original_context)
         return out
 
-    def exec_oc_debug_cmd(self, node, cmd_list, timeout=300, namespace=None):
+    def exec_oc_debug_cmd(
+        self, node, cmd_list, timeout=300, namespace=None, use_root=True
+    ):
         """
         Function to execute "oc debug" command on OCP node
 
@@ -258,11 +260,15 @@ def exec_oc_debug_cmd(self, node, cmd_list, timeout=300, namespace=None):
         create_cmd_list = copy.deepcopy(cmd_list)
         create_cmd_list.append(" ")
         err_msg = "CMD FAILED"
+        if use_root:
+            root_option = " chroot /host /bin/bash -c "
+        else:
+            root_option = " /bin/bash -c "
         cmd = f" || echo '{err_msg}';".join(create_cmd_list)
         namespace = namespace or config.ENV_DATA["cluster_namespace"]
         debug_cmd = (
             f"debug nodes/{node} --to-namespace={namespace} "
-            f' -- chroot /host /bin/bash -c "{cmd}"'
+            f' -- {root_option} "{cmd}"'
         )
         out = str(
             self.exec_oc_cmd(command=debug_cmd, out_yaml_format=False, timeout=timeout)
diff --git a/ocs_ci/ocs/resources/pod.py b/ocs_ci/ocs/resources/pod.py
index a66558b0479..14407a6df8d 100644
--- a/ocs_ci/ocs/resources/pod.py
+++ b/ocs_ci/ocs/resources/pod.py
@@ -4119,3 +4119,45 @@ def get_pods_pvcs(pod_objs, namespace=None):
     namespace = namespace or config.ENV_DATA["cluster_namespace"]
     pvc_names = [get_pvc_name(p) for p in pod_objs]
     return get_pvc_objs(pvc_names, namespace)
+
+
+def delete_pod_by_phase(
+    pod_phase,
+    namespace=config.ENV_DATA["cluster_namespace"],
+):
+    """
+    Delete the pods in a specific phase
+    Args:
+        pod_status (str): The pod status to delete
+        namespace (str): Name of cluster namespace(default: config.ENV_DATA["cluster_namespace"])
+    Returns:
+        bool: True, if the pods deleted successfully. False, otherwise
+    """
+    logger.info(f"Delete all the pods in the status '{pod_phase}'")
+    if pod_phase.lower() == "succeeded":
+        phase = "Succeeded"
+    elif pod_phase.lower() == "failed":
+        phase = "Failed"
+    elif pod_phase.lower() == "Pending":
+        phase = "Pending"
+    elif pod_phase.lower() == "running":
+        phase = "Running"
+    else:
+        raise ValueError(
+            f"Invalid pod status '{pod_phase}'. "
+            f"Valid options are 'succeeded' or 'failed'"
+        )
+
+    cmd = f"oc delete pod --field-selector=status.phase={phase} -n {namespace}"
+    logger.info(cmd)
+    try:
+        run_cmd(cmd=cmd)
+    except CommandFailed as ex:
+        logger.warning(
+            f"Failed to delete the pods in the status '{pod_phase}' due to the error: {ex}"
+        )
+        return False
+
+    logger.info(f"All '{pod_phase}' pods deleted successfully.")
+
+    return True
diff --git a/ocs_ci/resiliency/network_faults.py b/ocs_ci/resiliency/network_faults.py
index 045c6495a2c..d99c2eb0bd8 100644
--- a/ocs_ci/resiliency/network_faults.py
+++ b/ocs_ci/resiliency/network_faults.py
@@ -80,7 +80,7 @@ def _get_all_node_network_interfaces(self, interface_types):
                         iface.strip() for iface in output.splitlines() if iface.strip()
                     ]
                     interfaces.extend(ifaces)
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.error(f"Error retrieving interfaces from node {node.name}: {e}")
                     continue
 
@@ -160,7 +160,7 @@ def _apply_fault(self, description, netem_command):
                     cmd = f"tc qdisc del dev {iface} root"
                     try:
                         self.ocp_obj.exec_oc_debug_cmd(node=node.name, cmd_list=[cmd])
-                    except CommandFailed as e:
+                    except (CommandFailed, subprocess.TimeoutExpired) as e:
                         log.error(
                             f"Failed to remove fault from {node.name}/{iface}: {e}"
                         )
@@ -188,7 +188,7 @@ def _remove_faults_all_nodes(self):
                 try:
                     self.ocp_obj.exec_oc_debug_cmd(node=node.name, cmd_list=[cmd_del])
                     log.debug(f"Deleted qdisc on {node.name}/{iface}")
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.warning(f"Could not delete qdisc on {node.name}/{iface}: {e}")
                     continue
 
@@ -206,7 +206,7 @@ def _remove_faults_all_nodes(self):
                         log.info(
                             f"Verified: netem successfully removed from {node.name}/{iface}"
                         )
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.warning(
                         f"Could not verify qdisc status on {node.name}/{iface}: {e}"
                     )
diff --git a/ocs_ci/resiliency/node_stats.py b/ocs_ci/resiliency/node_stats.py
index 8aa6b1d30d9..309f47bc9d8 100644
--- a/ocs_ci/resiliency/node_stats.py
+++ b/ocs_ci/resiliency/node_stats.py
@@ -13,27 +13,63 @@ class NodeStats:
     """
 
     @staticmethod
-    def cpu_stats(node_obj):
+    def cpu_stats(node_obj, interval=1, count=2, format="json"):
         """
         Get CPU statistics for a given node using `mpstat`.
 
         Args:
-            node_obj (OCSNode): Node object to query.
+            node_obj (OCSNode): The node object to fetch stats from.
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
+            format (str): Output format - "json" or "text". Default "json".
 
         Returns:
-            list: List of CPU stats dictionaries parsed from mpstat JSON output.
+            list or str or None: Parsed statistics list (JSON), raw output (text),
+                                or None on failure.
         """
-        ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- mpstat 1 2 -o JSON"
+        log.info(
+            f"Running mpstat on node '{node_obj.name}' with interval={interval}, count={count}, format={format}"
+        )
+
+        if format not in ("json", "text"):
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return None
 
+        cmd = f"mpstat {interval} {count}"
+        if format == "json":
+            cmd += " -o JSON"
+        elif format == "text":
+            cmd += " > /tmp/mpstat.txt && cat /tmp/mpstat.txt"
+            log.warning("Text format selected. Output will be raw and unparsed.")
+
+        ocp_obj = ocp.OCP(kind="node")
         try:
-            log.info(f"Running mpstat on node: {node_obj.name}")
-            cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            return output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            cmd_output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False
+            )
+
+            if not cmd_output:
+                log.warning("Empty response received from mpstat command")
+                return None
+
+            if format == "json":
+                try:
+                    output = json.loads(cmd_output)
+                    return (
+                        output.get("sysstat", {})
+                        .get("hosts", [{}])[0]
+                        .get("statistics", [])
+                    )
+                except json.JSONDecodeError as e:
+                    log.error(
+                        f"Failed to parse JSON from mpstat on node '{node_obj.name}': {e}"
+                    )
+                    return None
+            return cmd_output.splitlines()
+
         except CommandFailed as e:
-            log.error(f"Failed to fetch CPU stats for node {node_obj.name}: {e}")
-            return []
+            log.error(f"Failed to fetch CPU stats from node '{node_obj.name}': {e}")
+            return None
 
     @staticmethod
     def memory_usage_percent(node_obj):
@@ -47,15 +83,20 @@ def memory_usage_percent(node_obj):
             float: Used memory percentage.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- cat /proc/meminfo"
+        cmd = "cat /proc/meminfo"
 
         try:
-            output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False, timeout=30
+            )
 
             meminfo = {}
             for line in output.splitlines():
-                key, value = line.strip().split(":", 1)
-                meminfo[key] = int(value.strip().split()[0])  # in KB
+                try:
+                    key, value = line.strip().split(":", 1)
+                    meminfo[key] = int(value.strip().split()[0])  # in KB
+                except (ValueError, IndexError):
+                    continue
 
             mem_total = meminfo.get("MemTotal")
             mem_available = meminfo.get("MemAvailable")
@@ -64,57 +105,88 @@ def memory_usage_percent(node_obj):
                 used_percent = ((mem_total - mem_available) / mem_total) * 100
                 log.info(f"Memory usage on node {node_obj.name}: {used_percent:.2f}%")
                 return round(used_percent, 2)
-            else:
-                log.warning("Missing MemTotal or MemAvailable in /proc/meminfo")
-                return 0.0
+
+            log.warning("Missing MemTotal or MemAvailable in /proc/meminfo")
+            return 0.0
 
         except CommandFailed as e:
-            log.error(f"Failed to compute memory usage on node {node_obj.name}: {e}")
+            log.error(
+                f"Failed to compute memory usage on node {node_obj.name}:  {str(e)}"
+            )
             return 0.0
 
     @staticmethod
-    def disk_stats(node_obj):
+    def disk_stats(node_obj, format="json", interval=1, count=2):
         """
         Get disk I/O statistics using `iostat`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            format (str): Output format ("json" or "text"). Default "json".
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
 
         Returns:
-            dict: Latest disk statistics.
+            dict or list: Latest disk statistics (dict for JSON, list for text).
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- iostat -xt -o JSON 1 2"
+        if format not in ("json", "text"):
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return {}
+
+        cmd = f"iostat -xt {interval} {count}"
+        if format == "json":
+            cmd += " -o JSON"
 
         try:
-            log.info(f"Running disk stats command on node: {node_obj.name}")
-            cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            stats = (
-                output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False, timeout=30
             )
-            return stats[-1] if stats else {}
+
+            if format == "json":
+                try:
+                    output = json.loads(output)
+                    stats = (
+                        output.get("sysstat", {})
+                        .get("hosts", [{}])[0]
+                        .get("statistics", [])
+                    )
+                    return stats[-1] if stats else {}
+                except json.JSONDecodeError as e:
+                    log.error(
+                        f"Failed to parse JSON from iostat on node '{node_obj.name}': {e}"
+                    )
+                    return {}
+
+            return output.splitlines()
+
         except CommandFailed as e:
-            log.error(f"Failed to fetch disk stats for node {node_obj.name}: {e}")
+            log.error(f"Failed to fetch disk stats from node '{node_obj.name}': {e}")
             return {}
 
     @staticmethod
-    def network_stats(node_obj, interface="ovn-k8s-mp0"):
+    def network_stats(node_obj, interface="ovn-k8s-mp0", interval=1, count=2):
         """
         Get network interface statistics using `sar`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            interface (str): Network interface to monitor. Default "ovn-k8s-mp0".
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
 
         Returns:
-            str: Network interface statistics as text.
+            list: Network interface statistics as text lines.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- sar -n DEV 1 1"
+        cmd = f"sar -n DEV {interval} {count}"
 
         try:
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False
+            )
             log.info(f"Running network stats command on node: {node_obj.name}")
-            return ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
+            return output.splitlines()
         except CommandFailed as e:
             log.error(f"Failed to fetch network stats for node {node_obj.name}: {e}")
-            return ""
+            return []
diff --git a/ocs_ci/resiliency/platform_failures.py b/ocs_ci/resiliency/platform_failures.py
index 08b83192488..613b0404452 100644
--- a/ocs_ci/resiliency/platform_failures.py
+++ b/ocs_ci/resiliency/platform_failures.py
@@ -17,6 +17,7 @@
 
 import logging
 import random
+import subprocess
 
 from ocs_ci.ocs.platform_nodes import PlatformNodesFactory
 from ocs_ci.resiliency.network_faults import NetworkFaults
@@ -148,7 +149,7 @@ def run_fault_simulation(nodes, interfaces, label):
                 nf = NetworkFaults(nodes, interface_types=interfaces)
                 nf.run()
                 log.info(f"[{label}] Completed fault simulation.")
-            except (ValueError, CommandFailed) as e:
+            except (ValueError, CommandFailed, subprocess.TimeoutExpired) as e:
                 log.error(f"[{label}] Error during simulation: {e}")
 
         # Helper to get a random subset of any list
diff --git a/ocs_ci/resiliency/platform_stress.py b/ocs_ci/resiliency/platform_stress.py
new file mode 100644
index 00000000000..a134b82fde7
--- /dev/null
+++ b/ocs_ci/resiliency/platform_stress.py
@@ -0,0 +1,299 @@
+import time
+import subprocess
+import random
+import threading
+import logging
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    NoRunningCephToolBoxException,
+)
+from ocs_ci.ocs import ocp
+from ocs_ci.utility.utils import ceph_health_check
+
+log = logging.getLogger(__name__)
+
+
+class PlatformStress:
+    """A class to perform stress testing on OpenShift cluster nodes using stress-ng."""
+
+    def __init__(self, nodes):
+        """Initializes PlatformStress with the given nodes.
+
+        Args:
+            nodes (list): List of node objects to perform stress testing on.
+        """
+        self.nodes = nodes
+        self.ocp_obj = ocp.OCP()
+        self.run_status = False  # Flag to control stress test execution
+        self.active_threads = []  # To keep track of active threads
+        self.stop_event = threading.Event()  # Event to signal threads to stop
+        log.info("Initialized PlatformStress with nodes: %s", [n.name for n in nodes])
+
+    def _apply_stress(self, node_obj, cmd_args, timeout=60, wait=True):
+        """Applies stress to a node using stress-ng with given arguments.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            cmd_args (str): Arguments to pass to stress-ng.
+            timeout (int, optional): Timeout in seconds for the stress command. Defaults to 60.
+            wait (bool, optional): Whether to wait for command completion. Defaults to True.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        success = True
+        full_cmd = f"stress-ng {cmd_args} --timeout {timeout}s"
+        log.info("[%s] Executing: %s", node_obj.name, full_cmd)
+        try:
+            self.ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name,
+                cmd_list=[full_cmd],
+                use_root=False,
+                timeout=timeout + 20,
+            )
+        except (
+            CommandFailed,
+            subprocess.TimeoutExpired,
+            NoRunningCephToolBoxException,
+        ) as e:
+            log.error("[%s] Stress command failed: %s", node_obj.name, e)
+            success = False
+
+        if wait:
+            log.debug("Waiting %s seconds for stress command to complete...", timeout)
+            # Check stop_event periodically instead of just sleeping
+            for _ in range(timeout):
+                if self.stop_event.is_set():
+                    log.info(
+                        "[%s] Stress test interrupted by stop event", node_obj.name
+                    )
+                    return False
+                time.sleep(1)
+        return success
+
+    def cpu_stress(self, node_obj, load_percentage=95, timeout=60):
+        """Applies CPU stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply CPU stress to.
+            load_percentage (int, optional): CPU load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        log.info(
+            "Running CPU stress test: %d processes at %d%% load",
+            cpu_count,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj,
+            f"--cpu {cpu_count} --cpu-load {load_percentage} --cpu-method matrixprod",
+            timeout,
+        )
+
+    def memory_stress(self, node_obj, process=1, load_percentage=95, timeout=120):
+        """Applies memory stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply memory stress to.
+            process (int, optional): Number of processes. Defaults to 1.
+            load_percentage (int, optional): Memory load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 120.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info(
+            "Running memory stress test: %d processes at %d%% memory",
+            process,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj, f"--vm {process} --vm-bytes {load_percentage}%", timeout
+        )
+
+    def io_stress(self, node_obj, process=8, timeout=60):
+        """Applies I/O stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply I/O stress to.
+            process (int, optional): Number of processes. Defaults to 8.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info("Running I/O stress test: %d processes", process)
+        return self._apply_stress(
+            node_obj,
+            f"--hdd {process} --hdd-bytes 4G --temp-path /mnt/disk1 --hdd-opts sync",
+            timeout,
+        )
+
+    def network_stress(self, node_obj, timeout=60):
+        """Applies network stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply network stress to.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        socket = cpu_count
+
+        if self.stop_event.is_set():
+            return False
+        log.info("Running network stress test: %d socket stressors", socket)
+        return self._apply_stress(node_obj, f"--sock {socket}", timeout)
+
+    def all_stress(self, node_obj, vm=1, io=4, timeout=60):
+        """Applies combined stress (CPU, memory, I/O, network) to a node.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            vm (int, optional): Number of VM stressors. Defaults to 1.
+            io (int, optional): Number of I/O stressors. Defaults to 4.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        sock = cpu_count
+        log.info(
+            "Running combined stress test: CPU=%d, VM=%d, IO=%d, SOCK=%d",
+            cpu_count,
+            vm,
+            io,
+            sock,
+        )
+        return self._apply_stress(
+            node_obj,
+            f" --cpu {cpu_count} --vm {vm} --iomix {io} --sock {sock} --temp-path /mnt/disk1 ",
+            timeout,
+        )
+
+    def _run_random_stress_loop(self, timeout=0, node_selection="ALL"):
+        """Internal method that runs in a loop to apply random stress tests.
+
+        Args:
+            timeout (int, optional): Timeout for each stress test. 0 means random.
+            node_selection (str, optional): "ALL" for all nodes or "RANDOM" for subset.
+        """
+        stress_funcs = [
+            self.cpu_stress,
+            self.memory_stress,
+            self.io_stress,
+            self.network_stress,
+            self.all_stress,
+        ]
+
+        while not self.stop_event.is_set():
+            if not self.nodes:
+                log.warning("No nodes available for stress test.")
+                break
+
+            subset_nodes = (
+                self.nodes
+                if node_selection.upper() == "ALL"
+                else random.sample(self.nodes, min(2, len(self.nodes)))
+            )
+            current_timeout = timeout or random.randint(30, 120)
+            selected = random.choice(stress_funcs)
+            log.info(
+                "Randomly selected %s for nodes: %s",
+                selected.__name__,
+                [n.name for n in subset_nodes],
+            )
+
+            for node in subset_nodes:
+                if self.stop_event.is_set():
+                    break
+                thread = threading.Thread(
+                    target=selected, args=(node,), kwargs={"timeout": current_timeout}
+                )
+                thread.start()
+                self.active_threads.append(thread)
+                log.info(
+                    "Started thread for %s on node: %s", selected.__name__, node.name
+                )
+
+            # Wait for current stress tests to complete or stop event
+            for thread in list(self.active_threads):
+                if self.stop_event.is_set():
+                    break
+                thread.join(timeout=1)
+                if not thread.is_alive():
+                    self.active_threads.remove(thread)
+
+            if not ceph_health_check(fix_ceph_health=True):
+                log.error("Ceph health check failed after scenario execution.")
+
+    def start_random_stress(self, timeout=0, node_selection="ALL"):
+        """Starts random stress tests in the background.
+
+        Args:
+            timeout (int, optional): Timeout for each stress test. 0 means random.
+            node_selection (str, optional): "ALL" for all nodes or "RANDOM" for subset.
+
+        Returns:
+            bool: True if stress test started successfully, False otherwise.
+        """
+        if self.run_status:
+            log.warning("Random stress test is already running")
+            return False
+
+        self.stop_event.clear()
+        self.run_status = True
+        self.background_thread = threading.Thread(
+            target=self._run_random_stress_loop,
+            args=(timeout, node_selection),
+            daemon=True,
+        )
+        self.background_thread.start()
+        log.info("Started random stress test in background")
+        return True
+
+    def stop(self):
+        """Stops all running stress tests.
+
+        Returns:
+            bool: True if stress tests were stopped successfully, False otherwise.
+        """
+        if not self.run_status:
+            log.warning("No stress test is currently running")
+            return False
+
+        self.stop_event.set()
+        self.run_status = False
+
+        # Wait for background thread to finish
+        if hasattr(self, "background_thread") and self.background_thread.is_alive():
+            self.background_thread.join(timeout=10)
+
+        # Wait for active stress threads to finish
+        for thread in list(self.active_threads):
+            thread.join(timeout=5)
+            if not thread.is_alive():
+                self.active_threads.remove(thread)
+
+        log.info("All stress tests have been stopped")
+        return True
+
+    def __del__(self):
+        """Cleans up when the object is deleted by stopping any running stress tests."""
+        if self.run_status:
+            self.stop()
diff --git a/ocs_ci/resiliency/resiliency_helper.py b/ocs_ci/resiliency/resiliency_helper.py
index 768e8ea2b7b..3619a1df843 100644
--- a/ocs_ci/resiliency/resiliency_helper.py
+++ b/ocs_ci/resiliency/resiliency_helper.py
@@ -31,7 +31,9 @@
     CommandFailed,
     CephHealthException,
     NoRunningCephToolBoxException,
+    TimeoutExpiredError,
 )
+from ocs_ci.ocs.resources.pod import delete_pod_by_phase
 
 log = logging.getLogger(__name__)
 
@@ -241,9 +243,23 @@ def post_scenario_check(self):
             log.error(f"Ceph crash logs found: {ceph_crashes}")
             raise Exception("Ceph crash logs found after scenario execution.")
         log.info("Checking Ceph health...")
-        if not ceph_health_check(fix_ceph_health=True):
+        if not ceph_health_check(fix_ceph_health=True, tries=25):
             log.error("Ceph health check failed after scenario execution.")
 
+        log.info("Ceph health check passed after scenario execution.")
+
+        # Removing Failed and Succeeded pods
+        # When we run node debug pods are created in the namespace
+        # and they are not deleted automatically.
+        delete_pod_by_phase(
+            "succeeded",
+            namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
+        )
+        delete_pod_by_phase(
+            "failed",
+            namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
+        )
+
     def start(self):
         """Start running all failure scenarios under this configuration."""
         for failure_case in self.resiliency_failures:
@@ -252,7 +268,10 @@ def start(self):
                 continue
             self.pre_scenario_check()
             log.info(f"Running failure case: {failure_case}")
-            self.inject_failure(failure_case)
+            try:
+                self.inject_failure(failure_case)
+            except (TimeoutExpiredError, CommandFailed, CephHealthException) as e:
+                log.error(f"Failure case execution failed: {e}")
             self.post_scenario_check()
 
     def inject_failure(self, failure):
diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py
index de0c5f33130..027ae1a1450 100644
--- a/ocs_ci/utility/utils.py
+++ b/ocs_ci/utility/utils.py
@@ -5014,8 +5014,12 @@ def get_ceph_crashes(toolbox_pod):
         list: List of ceph crash ID's
 
     """
-    ceph_crashes = toolbox_pod.exec_ceph_cmd("ceph crash ls")
-    return [each_crash["crash_id"] for each_crash in ceph_crashes]
+    try:
+        ceph_crashes = toolbox_pod.exec_ceph_cmd("ceph crash ls")
+        return [each_crash["crash_id"] for each_crash in ceph_crashes]
+    except (CommandFailed, subprocess.TimeoutExpired) as ex:
+        log.error(f"Failed to get ceph crash list: {ex}")
+        return []
 
 
 def archive_ceph_crashes(toolbox_pod):
diff --git a/tests/resiliency/conftest.py b/tests/resiliency/conftest.py
index d4ae75c46b4..e2e224c9e13 100644
--- a/tests/resiliency/conftest.py
+++ b/tests/resiliency/conftest.py
@@ -1,8 +1,13 @@
 import pytest
 import os
+import logging
 from ocs_ci.ocs import constants
 from ocs_ci.resiliency.resiliency_helper import ResiliencyConfig
 from ocs_ci.resiliency.resiliency_workload import workload_object
+from ocs_ci.resiliency.platform_stress import PlatformStress
+from ocs_ci.ocs.node import get_nodes
+
+log = logging.getLogger(__name__)
 
 
 @pytest.fixture
@@ -39,7 +44,7 @@ def factory(workload_type, pvc_obj, **kwargs):
         log_msg = f"Initializing resiliency workload: {workload_type}"
         if kwargs:
             log_msg += f" with args {kwargs}"
-        print(log_msg)
+        log.info(log_msg)
 
         # Instantiate the workload class (e.g., FioWorkload)
         workload_cls = workload_object(workload_type, namespace=pvc_obj.namespace)
@@ -53,3 +58,57 @@ def finalizer():
         return workload
 
     return factory
+
+
+@pytest.fixture
+def run_platform_stress(request):
+    """Factory fixture to create and run a PlatformStress object.
+
+    Automatically starts stress tests on given node types (default: worker nodes).
+    All stress will stop when the test completes.
+
+    Usage:
+        stress = run_platform_stress()
+        stress = run_platform_stress([constants.WORKER_MACHINE, constants.MASTER_MACHINE])
+
+    Returns:
+        function: Factory function to create PlatformStress instances.
+    """
+    created_instances = []
+
+    def factory(node_types=None):
+        """Creates and starts a PlatformStress instance.
+
+        Args:
+            node_types (list, optional): List of node types to include (e.g., WORKER_MACHINE, MASTER_MACHINE).
+
+        Returns:
+            PlatformStress: Initialized and running PlatformStress instance.
+        """
+        node_types = node_types or [constants.WORKER_MACHINE]
+        valid_types = {constants.WORKER_MACHINE, constants.MASTER_MACHINE}
+        if not set(node_types).issubset(valid_types):
+            unsupported = set(node_types) - valid_types
+            raise ValueError(f"Unsupported node types: {unsupported}")
+
+        nodes = [node for nt in node_types for node in get_nodes(nt)]
+        log.info(
+            "Creating PlatformStress instance for nodes: %s", [n.name for n in nodes]
+        )
+
+        stress_obj = PlatformStress(nodes)
+        stress_obj.start_random_stress()
+        created_instances.append(stress_obj)
+        log.info("Started stress testing in background.")
+        return stress_obj
+
+    def finalizer():
+        """Cleanup function to stop all PlatformStress instances."""
+        for stress_obj in created_instances:
+            if stress_obj.run_status:
+                log.info("Stopping stress for PlatformStress instance...")
+                stress_obj.stop()
+                log.info("Stress stopped.")
+
+    request.addfinalizer(finalizer)
+    return factory
diff --git a/tests/resiliency/test_platfrom_failures_scenarios.py b/tests/resiliency/test_platfrom_failures_scenarios.py
index 277fd180fe6..e142ec02c62 100644
--- a/tests/resiliency/test_platfrom_failures_scenarios.py
+++ b/tests/resiliency/test_platfrom_failures_scenarios.py
@@ -10,14 +10,6 @@
 
 @green_squad
 @resiliency
-@pytest.mark.parametrize(
-    "failure_case",
-    [
-        "PLATFORM_INSTANCE_FAILURES",
-        "PLATFORM_NETWORK_FAILURES",
-        "PLATFORM_NETWORK_FAULTS",
-    ],
-)
 class TestPlatformFailureScenarios:
     def _prepare_pvcs_and_workloads(
         self, project_factory, multi_pvc_factory, resiliency_workload
@@ -67,6 +59,14 @@ def _validate_and_cleanup_workloads(self, workloads):
 
         log.info("All workloads passed after failure injection.")
 
+    @pytest.mark.parametrize(
+        "failure_case",
+        [
+            "PLATFORM_INSTANCE_FAILURES",
+            "PLATFORM_NETWORK_FAILURES",
+            "PLATFORM_NETWORK_FAULTS",
+        ],
+    )
     def test_platform_failure_scenarios(
         self,
         failure_case,
@@ -94,3 +94,39 @@ def test_platform_failure_scenarios(
         resiliency_runner.cleanup()
 
         self._validate_and_cleanup_workloads(workloads)
+
+    @pytest.mark.parametrize(
+        "failure_case",
+        [
+            "PLATFORM_INSTANCE_FAILURES",
+            "PLATFORM_NETWORK_FAILURES",
+        ],
+    )
+    def test_platform_failures_with_stress(
+        self,
+        failure_case,
+        platfrom_failure_scenarios,
+        project_factory,
+        multi_pvc_factory,
+        resiliency_workload,
+        run_platform_stress,
+    ):
+        """
+        Validates platform resiliency under stress conditions
+        like high CPU, memory, I/O, and network load.
+        Ensures workloads continue running during failure scenarios.
+        """
+        scenario = platfrom_failure_scenarios.get("SCENARIO_NAME")
+        log.info(f"Running Scenario: {scenario}, Failure Case: {failure_case}")
+
+        workloads = self._prepare_pvcs_and_workloads(
+            project_factory, multi_pvc_factory, resiliency_workload
+        )
+
+        run_platform_stress()
+
+        resiliency_runner = Resiliency(scenario, failure_method=failure_case)
+        resiliency_runner.start()
+        resiliency_runner.cleanup()
+
+        self._validate_and_cleanup_workloads(workloads)


Reviews:
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           pakamble-418vs8
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           /tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress[PLATFORM_INSTANCE_FAILURES]
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](https://url.corp.redhat.com/6a49d49) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T09:50:52Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           pakamble-418vs8
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           /tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](https://url.corp.redhat.com/b012c9b) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T12:37:53Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           pakamble-418vs8
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/d10943f) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-20T09:13:01Z
user: login: ebenahar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-20T16:13:29Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           pakamble-418vs9
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/793461e) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-20T18:15:27Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/b64351d) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-21T00:57:57Z
user: login: paraggit
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T12:48:26Z
user: login: paraggit
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T12:49:01Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/bccfa6b) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-21T18:02:33Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](https://url.corp.redhat.com/53790ff) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-21T20:25:04Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           pakamble-418vs8
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/d2fc338) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T02:13:55Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8663//testReport) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T07:46:52Z
user: login: AviadP
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T12:21:58Z
user: login: AviadP
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T12:23:30Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job UNSTABLE](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8664//testReport) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T16:37:27Z
user: login: paraggit
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T17:37:46Z
user: login: paraggit
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T17:48:35Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8665//testReport) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T18:36:29Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8667//testReport) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-23T02:53:35Z
user: login: AviadP
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T06:28:09Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job FAILED](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8668//testReport) (installation failed, tests not executed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-23T07:02:02Z
user: login: ocs-ci
body: PR validation 


Cluster Name:           
Cluster Configuration:  
PR Test Suite:          
PR Test Path:           tests/resiliency/test_platfrom_failures_scenarios.py::TestPlatformFailureScenarios::test_platform_failures_with_stress
Additional Test Params: 
OCP VERSION:            4.18
OCS VERSION:            4.18
tested against branch:  master


 [Job PASSED](http://SHORTENED-URL-ERROR/job/qe-trigger-test-pr/8670//testReport).
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-23T09:21:58Z
user: login: ebenahar
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T09:32:22Z

Review comments:
diff_hunk: @@ -13,27 +13,54 @@ class NodeStats:
     """
 
     @staticmethod
-    def cpu_stats(node_obj):
+    def cpu_stats(node_obj, format="json", interval=1, count=2):
         """
         Get CPU statistics for a given node using `mpstat`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            format (str): Output format ('json' or 'text').
+            interval (int): Interval between samples in seconds.
+            count (int): Number of samples to collect.
 
         Returns:
-            list: List of CPU stats dictionaries parsed from mpstat JSON output.
+            list or None: List of CPU stats dictionaries (if format is 'json'),
+            or None for unsupported format.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- mpstat 1 2 -o JSON"
+
+        if format == "json":
+            cmd = f"debug nodes/{node_obj.name} -- mpstat {interval} {count} -o JSON"
+        elif format == "text":
+            cmd = f"debug nodes/{node_obj.name} -- mpstat {interval} {count} > /tmp/mpstat.txt"
+            log.warning(
+                "Text format selected. This method currently does not return parsed text output."
+            )
+            return None
+        else:
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return None
 
         try:
-            log.info(f"Running mpstat on node: {node_obj.name}")
+            log.info(
+                f"Running mpstat on node '{node_obj.name}' with interval={interval}, count={count}"
+            )
             cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            return output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            if format == "json":
+                output = json.loads(cmd_output)
+                return (
+                    output.get("sysstat", {})
+                    .get("hosts", [{}])[0]
+                    .get("statistics", [])
+                )
         except CommandFailed as e:
-            log.error(f"Failed to fetch CPU stats for node {node_obj.name}: {e}")
-            return []
+            log.error(f"Failed to fetch CPU stats for node '{node_obj.name}': {e}")
+        except json.JSONDecodeError as e:
+            log.error(
+                f"Failed to parse JSON output from mpstat on node '{node_obj.name}': {e}"
+            )
+
+        return []
path: ocs_ci/resiliency/node_stats.py
user: login: ebenahar
body: Wouldn't it be better to move this code under line 32?
created_at: 2025-04-20T15:58:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,298 @@
+import time
+import subprocess
+import random
+import threading
+import logging
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    NoRunningCephToolBoxException,
+)
+from ocs_ci.ocs import ocp
+
+log = logging.getLogger(__name__)
+
+
+class PlatformStress:
path: ocs_ci/resiliency/platform_stress.py
user: login: ebenahar
body: @paraggit @PrasadDesala @mashetty330 please coordinate this with the stress automation 
created_at: 2025-04-20T16:08:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,298 @@
+import time
+import subprocess
+import random
+import threading
+import logging
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    NoRunningCephToolBoxException,
+)
+from ocs_ci.ocs import ocp
+
+log = logging.getLogger(__name__)
+
+
+class PlatformStress:
+    """A class to perform stress testing on OpenShift cluster nodes using stress-ng."""
+
+    def __init__(self, nodes):
+        """Initializes PlatformStress with the given nodes.
+
+        Args:
+            nodes (list): List of node objects to perform stress testing on.
+        """
+        self.nodes = nodes
+        self.ocp_obj = ocp.OCP()
+        self.run_status = False  # Flag to control stress test execution
+        self.active_threads = []  # To keep track of active threads
+        self.stop_event = threading.Event()  # Event to signal threads to stop
+        log.info("Initialized PlatformStress with nodes: %s", [n.name for n in nodes])
+
+    def _apply_stress(self, node_obj, cmd_args, timeout=60, wait=True):
+        """Applies stress to a node using stress-ng with given arguments.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            cmd_args (str): Arguments to pass to stress-ng.
+            timeout (int, optional): Timeout in seconds for the stress command. Defaults to 60.
+            wait (bool, optional): Whether to wait for command completion. Defaults to True.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        success = True
+        full_cmd = f"stress-ng {cmd_args} --timeout {timeout}s"
+        log.info("[%s] Executing: %s", node_obj.name, full_cmd)
+        try:
+            self.ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name,
+                cmd_list=[full_cmd],
+                use_root=False,
+                timeout=timeout + 20,
+            )
+        except (
+            CommandFailed,
+            subprocess.TimeoutExpired,
+            NoRunningCephToolBoxException,
+        ) as e:
+            log.error("[%s] Stress command failed: %s", node_obj.name, e)
+            success = False
+
+        if wait:
+            log.debug("Waiting %s seconds for stress command to complete...", timeout)
+            # Check stop_event periodically instead of just sleeping
+            for _ in range(timeout):
+                if self.stop_event.is_set():
+                    log.info(
+                        "[%s] Stress test interrupted by stop event", node_obj.name
+                    )
+                    return False
+                time.sleep(1)
+        return success
+
+    def cpu_stress(self, node_obj, load_percentage=95, timeout=60):
+        """Applies CPU stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply CPU stress to.
+            load_percentage (int, optional): CPU load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        log.info(
+            "Running CPU stress test: %d processes at %d%% load",
+            cpu_count,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj,
+            f"--cpu {cpu_count} --cpu-load {load_percentage} --cpu-method matrixprod",
+            timeout,
+        )
+
+    def memory_stress(self, node_obj, process=1, load_percentage=95, timeout=120):
+        """Applies memory stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply memory stress to.
+            process (int, optional): Number of processes. Defaults to 1.
+            load_percentage (int, optional): Memory load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 120.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info(
+            "Running memory stress test: %d processes at %d%% memory",
+            process,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj, f"--vm {process} --vm-bytes {load_percentage}%", timeout
+        )
+
+    def io_stress(self, node_obj, process=8, timeout=60):
+        """Applies I/O stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply I/O stress to.
+            process (int, optional): Number of processes. Defaults to 8.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info("Running I/O stress test: %d processes", process)
+        return self._apply_stress(
+            node_obj,
+            f"--hdd {process} --hdd-bytes 4G --temp-path /mnt/disk1 --hdd-opts sync",
+            timeout,
+        )
+
+    def network_stress(self, node_obj, timeout=60):
+        """Applies network stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply network stress to.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        socket = cpu_count
+
+        if self.stop_event.is_set():
+            return False
+        log.info("Running network stress test: %d socket stressors", socket)
+        return self._apply_stress(node_obj, f"--sock {socket}", timeout)
+
+    def all_stress(self, node_obj, vm=1, io=4, timeout=60):
+        """Applies combined stress (CPU, memory, I/O, network) to a node.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            vm (int, optional): Number of VM stressors. Defaults to 1.
+            io (int, optional): Number of I/O stressors. Defaults to 4.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        sock = cpu_count
+        log.info(
+            "Running combined stress test: CPU=%d, VM=%d, IO=%d, SOCK=%d",
+            cpu_count,
+            vm,
+            io,
+            sock,
+        )
+        return self._apply_stress(
+            node_obj,
+            f" --cpu {cpu_count} --vm {vm} --iomix {io} --sock {sock} --temp-path /mnt/disk1 ",
+            timeout,
+        )
+
+    def _run_random_stress_loop(self, timeout=0, node_selection="ALL"):
+        """Internal method that runs in a loop to apply random stress tests.
+
+        Args:
+            timeout (int, optional): Timeout for each stress test. 0 means random.
+            node_selection (str, optional): "ALL" for all nodes or "RANDOM" for subset.
+        """
+        stress_funcs = [
+            self.cpu_stress,
+            self.memory_stress,
+            self.io_stress,
+            self.network_stress,
+            self.all_stress,
+        ]
+
+        while not self.stop_event.is_set():
+            if not self.nodes:
+                log.warning("No nodes available for stress test.")
+                break
+
+            subset_nodes = (
+                self.nodes
+                if node_selection.upper() == "ALL"
+                else random.sample(self.nodes, min(2, len(self.nodes)))
+            )
+            current_timeout = timeout or random.randint(30, 120)
+            selected = random.choice(stress_funcs)
+            log.info(
+                "Randomly selected %s for nodes: %s",
+                selected.__name__,
+                [n.name for n in subset_nodes],
+            )
+
+            for node in subset_nodes:
+                if self.stop_event.is_set():
+                    break
+                thread = threading.Thread(
+                    target=selected, args=(node,), kwargs={"timeout": current_timeout}
+                )
+                thread.start()
+                self.active_threads.append(thread)
+                log.info(
+                    "Started thread for %s on node: %s", selected.__name__, node.name
+                )
+
+            # Wait for current stress tests to complete or stop event
+            for thread in list(self.active_threads):
+                if self.stop_event.is_set():
+                    break
+                thread.join(timeout=1)
+                if not thread.is_alive():
+                    self.active_threads.remove(thread)
+
+            log.info("Cooling down for 30 seconds before next stress test...")
path: ocs_ci/resiliency/platform_stress.py
user: login: ebenahar
body: if possible, let's avoid sleep and use some health validation 
created_at: 2025-04-20T16:13:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,27 +13,54 @@ class NodeStats:
     """
 
     @staticmethod
-    def cpu_stats(node_obj):
+    def cpu_stats(node_obj, format="json", interval=1, count=2):
         """
         Get CPU statistics for a given node using `mpstat`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            format (str): Output format ('json' or 'text').
+            interval (int): Interval between samples in seconds.
+            count (int): Number of samples to collect.
 
         Returns:
-            list: List of CPU stats dictionaries parsed from mpstat JSON output.
+            list or None: List of CPU stats dictionaries (if format is 'json'),
+            or None for unsupported format.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- mpstat 1 2 -o JSON"
+
+        if format == "json":
+            cmd = f"debug nodes/{node_obj.name} -- mpstat {interval} {count} -o JSON"
+        elif format == "text":
+            cmd = f"debug nodes/{node_obj.name} -- mpstat {interval} {count} > /tmp/mpstat.txt"
+            log.warning(
+                "Text format selected. This method currently does not return parsed text output."
+            )
+            return None
+        else:
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return None
 
         try:
-            log.info(f"Running mpstat on node: {node_obj.name}")
+            log.info(
+                f"Running mpstat on node '{node_obj.name}' with interval={interval}, count={count}"
+            )
             cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            return output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            if format == "json":
+                output = json.loads(cmd_output)
+                return (
+                    output.get("sysstat", {})
+                    .get("hosts", [{}])[0]
+                    .get("statistics", [])
+                )
         except CommandFailed as e:
-            log.error(f"Failed to fetch CPU stats for node {node_obj.name}: {e}")
-            return []
+            log.error(f"Failed to fetch CPU stats for node '{node_obj.name}': {e}")
+        except json.JSONDecodeError as e:
+            log.error(
+                f"Failed to parse JSON output from mpstat on node '{node_obj.name}': {e}"
+            )
+
+        return []
path: ocs_ci/resiliency/node_stats.py
user: login: paraggit
body: Corrected the code
created_at: 2025-04-21T12:48:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,298 @@
+import time
+import subprocess
+import random
+import threading
+import logging
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    NoRunningCephToolBoxException,
+)
+from ocs_ci.ocs import ocp
+
+log = logging.getLogger(__name__)
+
+
+class PlatformStress:
+    """A class to perform stress testing on OpenShift cluster nodes using stress-ng."""
+
+    def __init__(self, nodes):
+        """Initializes PlatformStress with the given nodes.
+
+        Args:
+            nodes (list): List of node objects to perform stress testing on.
+        """
+        self.nodes = nodes
+        self.ocp_obj = ocp.OCP()
+        self.run_status = False  # Flag to control stress test execution
+        self.active_threads = []  # To keep track of active threads
+        self.stop_event = threading.Event()  # Event to signal threads to stop
+        log.info("Initialized PlatformStress with nodes: %s", [n.name for n in nodes])
+
+    def _apply_stress(self, node_obj, cmd_args, timeout=60, wait=True):
+        """Applies stress to a node using stress-ng with given arguments.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            cmd_args (str): Arguments to pass to stress-ng.
+            timeout (int, optional): Timeout in seconds for the stress command. Defaults to 60.
+            wait (bool, optional): Whether to wait for command completion. Defaults to True.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        success = True
+        full_cmd = f"stress-ng {cmd_args} --timeout {timeout}s"
+        log.info("[%s] Executing: %s", node_obj.name, full_cmd)
+        try:
+            self.ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name,
+                cmd_list=[full_cmd],
+                use_root=False,
+                timeout=timeout + 20,
+            )
+        except (
+            CommandFailed,
+            subprocess.TimeoutExpired,
+            NoRunningCephToolBoxException,
+        ) as e:
+            log.error("[%s] Stress command failed: %s", node_obj.name, e)
+            success = False
+
+        if wait:
+            log.debug("Waiting %s seconds for stress command to complete...", timeout)
+            # Check stop_event periodically instead of just sleeping
+            for _ in range(timeout):
+                if self.stop_event.is_set():
+                    log.info(
+                        "[%s] Stress test interrupted by stop event", node_obj.name
+                    )
+                    return False
+                time.sleep(1)
+        return success
+
+    def cpu_stress(self, node_obj, load_percentage=95, timeout=60):
+        """Applies CPU stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply CPU stress to.
+            load_percentage (int, optional): CPU load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        log.info(
+            "Running CPU stress test: %d processes at %d%% load",
+            cpu_count,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj,
+            f"--cpu {cpu_count} --cpu-load {load_percentage} --cpu-method matrixprod",
+            timeout,
+        )
+
+    def memory_stress(self, node_obj, process=1, load_percentage=95, timeout=120):
+        """Applies memory stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply memory stress to.
+            process (int, optional): Number of processes. Defaults to 1.
+            load_percentage (int, optional): Memory load percentage. Defaults to 95.
+            timeout (int, optional): Timeout in seconds. Defaults to 120.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info(
+            "Running memory stress test: %d processes at %d%% memory",
+            process,
+            load_percentage,
+        )
+        return self._apply_stress(
+            node_obj, f"--vm {process} --vm-bytes {load_percentage}%", timeout
+        )
+
+    def io_stress(self, node_obj, process=8, timeout=60):
+        """Applies I/O stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply I/O stress to.
+            process (int, optional): Number of processes. Defaults to 8.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        log.info("Running I/O stress test: %d processes", process)
+        return self._apply_stress(
+            node_obj,
+            f"--hdd {process} --hdd-bytes 4G --temp-path /mnt/disk1 --hdd-opts sync",
+            timeout,
+        )
+
+    def network_stress(self, node_obj, timeout=60):
+        """Applies network stress to a node.
+
+        Args:
+            node_obj (object): The node object to apply network stress to.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        socket = cpu_count
+
+        if self.stop_event.is_set():
+            return False
+        log.info("Running network stress test: %d socket stressors", socket)
+        return self._apply_stress(node_obj, f"--sock {socket}", timeout)
+
+    def all_stress(self, node_obj, vm=1, io=4, timeout=60):
+        """Applies combined stress (CPU, memory, I/O, network) to a node.
+
+        Args:
+            node_obj (object): The node object to apply stress to.
+            vm (int, optional): Number of VM stressors. Defaults to 1.
+            io (int, optional): Number of I/O stressors. Defaults to 4.
+            timeout (int, optional): Timeout in seconds. Defaults to 60.
+
+        Returns:
+            bool: True if stress command succeeded, False otherwise.
+        """
+        if self.stop_event.is_set():
+            return False
+        cpu_count = int(node_obj.data["status"]["capacity"]["cpu"])
+        sock = cpu_count
+        log.info(
+            "Running combined stress test: CPU=%d, VM=%d, IO=%d, SOCK=%d",
+            cpu_count,
+            vm,
+            io,
+            sock,
+        )
+        return self._apply_stress(
+            node_obj,
+            f" --cpu {cpu_count} --vm {vm} --iomix {io} --sock {sock} --temp-path /mnt/disk1 ",
+            timeout,
+        )
+
+    def _run_random_stress_loop(self, timeout=0, node_selection="ALL"):
+        """Internal method that runs in a loop to apply random stress tests.
+
+        Args:
+            timeout (int, optional): Timeout for each stress test. 0 means random.
+            node_selection (str, optional): "ALL" for all nodes or "RANDOM" for subset.
+        """
+        stress_funcs = [
+            self.cpu_stress,
+            self.memory_stress,
+            self.io_stress,
+            self.network_stress,
+            self.all_stress,
+        ]
+
+        while not self.stop_event.is_set():
+            if not self.nodes:
+                log.warning("No nodes available for stress test.")
+                break
+
+            subset_nodes = (
+                self.nodes
+                if node_selection.upper() == "ALL"
+                else random.sample(self.nodes, min(2, len(self.nodes)))
+            )
+            current_timeout = timeout or random.randint(30, 120)
+            selected = random.choice(stress_funcs)
+            log.info(
+                "Randomly selected %s for nodes: %s",
+                selected.__name__,
+                [n.name for n in subset_nodes],
+            )
+
+            for node in subset_nodes:
+                if self.stop_event.is_set():
+                    break
+                thread = threading.Thread(
+                    target=selected, args=(node,), kwargs={"timeout": current_timeout}
+                )
+                thread.start()
+                self.active_threads.append(thread)
+                log.info(
+                    "Started thread for %s on node: %s", selected.__name__, node.name
+                )
+
+            # Wait for current stress tests to complete or stop event
+            for thread in list(self.active_threads):
+                if self.stop_event.is_set():
+                    break
+                thread.join(timeout=1)
+                if not thread.is_alive():
+                    self.active_threads.remove(thread)
+
+            log.info("Cooling down for 30 seconds before next stress test...")
path: ocs_ci/resiliency/platform_stress.py
user: login: paraggit
body: Added ceph health check call 
created_at: 2025-04-21T12:49:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,7 +80,7 @@ def _get_all_node_network_interfaces(self, interface_types):
                         iface.strip() for iface in output.splitlines() if iface.strip()
                     ]
                     interfaces.extend(ifaces)
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
path: ocs_ci/resiliency/network_faults.py
user: login: AviadP
body: You catch subprocess.TimeoutExpired but only log and continue; consider a retry/back‑off strategy if transient or a cleanup function in failure case, to make sure that network recovered
created_at: 2025-04-22T12:21:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -4119,3 +4119,43 @@ def get_pods_pvcs(pod_objs, namespace=None):
     namespace = namespace or config.ENV_DATA["cluster_namespace"]
     pvc_names = [get_pvc_name(p) for p in pod_objs]
     return get_pvc_objs(pvc_names, namespace)
+
+
+def delete_pod_by_phase(
path: ocs_ci/ocs/resources/pod.py
user: login: AviadP
body: consider add log statement on success ```logger.info(f"All '{pod_phase}' pods deleted successfully")```

created_at: 2025-04-22T12:23:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -4119,3 +4119,43 @@ def get_pods_pvcs(pod_objs, namespace=None):
     namespace = namespace or config.ENV_DATA["cluster_namespace"]
     pvc_names = [get_pvc_name(p) for p in pod_objs]
     return get_pvc_objs(pvc_names, namespace)
+
+
+def delete_pod_by_phase(
path: ocs_ci/ocs/resources/pod.py
user: login: paraggit
body: Added
created_at: 2025-04-22T17:37:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,7 +80,7 @@ def _get_all_node_network_interfaces(self, interface_types):
                         iface.strip() for iface in output.splitlines() if iface.strip()
                     ]
                     interfaces.extend(ifaces)
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
path: ocs_ci/resiliency/network_faults.py
user: login: paraggit
body: When stress is applied to the system the command might take too long and cause a `TimeoutExpired` exception. But since we already reboot the nodes after injecting faults, that will automatically clear all the stress. So I’ve handled the timeout here because I don’t want the code to stop because of it.
created_at: 2025-04-22T17:48:35Z
author_association: CONTRIBUTOR

PR comments:
user: login: openshift-ci[bot]
created_at: 2025-04-23T09:32:28Z
body: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11925#pullrequestreview-2786076761" title="Approved">AviadP</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11925#pullrequestreview-2786636367" title="Approved">ebenahar</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/11925#" title="Author self-approved">paraggit</a>*


The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=red-hat-storage%2Focs-ci).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/red-hat-storage/ocs-ci/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE