Repository: sonic-net/sonic-mgmt
Pull request #17612: DPU Parallel power operations in Tests
Author: login: nissampa
Created: 2025-03-19T22:45:29Z | Merged: 2025-04-02T02:31:44Z

This is the improvement of the test cases represented in this PR: 
https://github.com/sonic-net/sonic-mgmt/pull/16020

It is tested on Cisco smart switch platform. 


Diff:
diff --git a/tests/smartswitch/common/device_utils_dpu.py b/tests/smartswitch/common/device_utils_dpu.py
index 526ec742adc..0fe8b9303d5 100644
--- a/tests/smartswitch/common/device_utils_dpu.py
+++ b/tests/smartswitch/common/device_utils_dpu.py
@@ -12,6 +12,7 @@
      import check_interface_status_of_up_ports
 from tests.common.reboot import wait_for_startup
 from tests.common.platform.processes_utils import wait_critical_processes
+from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
 
 # Timeouts, Delays and Time Intervals in secs
 DPU_TIMEOUT = 210
@@ -69,7 +70,7 @@ def is_dark_mode_enabled(duthost, platform_api_conn, num_dpu_modules):   # noqa
     """
     Checks the liveliness of DPU
     Returns:
-        True if all DPUs admin status are down
+        True if all DPUs admin status are DOWN
         else False
     """
 
@@ -112,7 +113,7 @@ def dpu_power_on(duthost, platform_api_conn, num_dpu_modules):    # noqa F811
     pytest_assert(wait_until(PING_MAX_TIMEOUT, PING_MAX_TIME_INT, 0,
                   check_dpu_ping_status,
                   duthost, ip_address_list),
-                  "Not all DPUs are operationally up")
+                  "Not all DPUs are operationally UP")
 
 
 def check_dpu_ping_status(duthost, ip_address_list):
@@ -139,10 +140,10 @@ def check_dpu_ping_status(duthost, ip_address_list):
 
 def check_dpu_module_status(duthost, power_status, dpu_name):
     """
-    Check status of given DPU module against given option on/off
+    Check status of given DPU module against given option ON/OFF
     Args:
         duthost : Host handle
-        power_status: on/off status of dpu
+        power_status: ON/OFF status of dpu
         dpu_name: name of the dpu module
     Returns:
         Returns True or False based on status of given DPU module
@@ -200,10 +201,10 @@ def check_pmon_status(duthost):
     """
     output_pmon_status = duthost.shell('docker ps | grep pmon')
     if "up" in output_pmon_status['stdout'].lower():
-        logging.info("pmon container is up")
+        logging.info("pmon container is UP")
         return True
 
-    logging.error("pmon container is not up")
+    logging.error("pmon container is not UP")
     return False
 
 
@@ -272,13 +273,13 @@ def check_dpu_link_and_status(duthost, dpu_on_list,
         pytest_assert(wait_until(DPU_TIMEOUT, DPU_TIME_INT, 0,
                       check_dpu_module_status,
                       duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up")
+                      "DPU is not operationally UP")
 
     for index in range(len(dpu_off_list)):
         pytest_assert(wait_until(DPU_TIMEOUT, DPU_TIME_INT, 0,
                       check_dpu_module_status,
                       duthost, "off", dpu_off_list[index]),
-                      "DPU is not operationally down")
+                      "DPU is not operationally DOWN")
 
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to DPU has failed")
@@ -345,27 +346,30 @@ def check_dpu_health_status(duthost, dpu_name,
     return
 
 
-def check_dpu_critical_processes(dpuhosts, dpu_number):
+def check_dpu_critical_processes(dpuhosts, dpu_id):
 
     """
-    Checks all critical processes are up on DPU
+    Checks all critical processes are UP on DPU
     If not, fails the case
     Args:
        dpuhosts: DPU Host handle
-       num_dpu_modules: Gets number of DPU modules
+       dpu_id: DPU ID
     Returns:
        Nothing
     """
 
     cmd = "sudo show system-health detail"
-    output_dpu_process = dpuhosts[dpu_number].show_and_parse(cmd)
+    output_dpu_process = dpuhosts[dpu_id].show_and_parse(cmd)
 
     for index in range(len(output_dpu_process)):
         parse_output = output_dpu_process[index]
-        pytest_assert(parse_output['status'].lower() == 'ok',
-                      f"{parse_output['name']} not Ok in DPU{dpu_number}")
-
-    return
+        if parse_output['status'].lower() == 'ok':
+            continue
+        else:
+            logging.error("'{}' has failed in DPU{}"
+                          .format(parse_output["name"], dpu_id))
+            return False
+    return True
 
 
 def pre_test_check(duthost,
@@ -416,8 +420,8 @@ def post_test_switch_check(duthost, localhost,
     logging.info("Checking for Interface status")
     pytest_assert(wait_until(INTF_MAX_TIMEOUT, INTF_TIME_INT, 0,
                   check_interface_status_of_up_ports, duthost),
-                  "Not all ports that are admin up, are operationally up")
-    logging.info("Interfaces are up")
+                  "Not all ports that are admin up, are operationally UP")
+    logging.info("Interfaces are UP")
 
     logging.info("Wait until all critical services are fully started")
     wait_critical_processes(duthost)
@@ -429,46 +433,114 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
-def post_test_dpu_check(duthost, dpuhosts,
-                        dpu_on_list, dpu_off_list,
-                        ip_address_list):
+def post_test_dpu_check(duthost, dpuhosts, dpu_name):
     """
-    Checks DPU off/on and reboot cause status Post Test
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is UP post test")
+    pytest_assert(
+        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+                   check_dpu_module_status, duthost, "on", dpu_name),
+        f"DPU {dpu_name} is not operationally UP post the operation"
+    )
+
+    dpu_id = int(re.search(r'\d+', dpu_name).group())
+    logging.info(f"Checking critical processes on {dpu_name}")
+    pytest_assert(
+        wait_until(
+            DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+            check_dpu_critical_processes, dpuhosts, dpu_id),
+        f"Crictical process check for {dpu_name} has been failed"
+    )
+
+    logging.info(f"Checking reboot cause of {dpu_name}")
+    pytest_assert(
+        wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
+                   check_dpu_reboot_cause, duthost, dpu_name, "Non-Hardware"),
+        f"Reboot cause for DPU {dpu_name} is incorrect"
+    )
+
+
+def post_test_dpus_check(duthost, dpuhosts,
+                         dpu_on_list, dpu_off_list,
+                         ip_address_list, num_dpu_modules):
+    """
+    Checks DPU OFF/ON and reboot cause status Post Test
     Args:
        duthost: Host handle
        dpuhosts: DPU Host handle
        dpu_on_list: List of DPUs that are ON
        dpu_off_list: List of DPUs that are OFF
        ip_address_list: List of DPU IP address that are ON
+       num_dpu_modules: number of dpu modules
     Returns:
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Post test DPUs check in parallel")
+        for dpu in dpu_on_list:
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu)
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list, num_dpu_modules):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+       num_dpu_modules: number of dpu modules
+    Returns:
+       Returns Nothing
+    """
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Check shutdown of DPUs in parallel")
+        for dpu_name in dpu_list:
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            )
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "off", dpu_name
+            )
+
+
+def dpus_startup_and_check(duthost, dpu_list, num_dpu_modules):
+    """
+    Parallely Execute DPU startup for given DPU list
+    Waits and checks parallely whether DPU is actually UP
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be startup
+
+    Returns:
+       Returns Nothing
+    """
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Check startup of DPUs in parallel")
+        for dpu_name in dpu_list:
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules startup {dpu_name}"
+            )
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "on", dpu_name
+            )
diff --git a/tests/smartswitch/platform_tests/test_platform_dpu.py b/tests/smartswitch/platform_tests/test_platform_dpu.py
index e924dfdfc6b..b7deb660830 100644
--- a/tests/smartswitch/platform_tests/test_platform_dpu.py
+++ b/tests/smartswitch/platform_tests/test_platform_dpu.py
@@ -11,9 +11,11 @@
 from tests.smartswitch.common.device_utils_dpu import check_dpu_ping_status,\
     check_dpu_module_status, check_dpu_reboot_cause, check_pmon_status,\
     parse_dpu_memory_usage, parse_system_health_summary,\
-    pre_test_check, post_test_dpu_check,\
+    pre_test_check, post_test_dpus_check,\
+    dpus_shutdown_and_check, dpus_startup_and_check,\
     check_dpu_health_status, num_dpu_modules  # noqa: F401
 from tests.common.platform.device_utils import platform_api_conn, start_platform_api_service  # noqa: F401,F403
+from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
 
 pytestmark = [
     pytest.mark.topology('smartswitch')
@@ -22,7 +24,6 @@
 # Timeouts, Delays and Time Intervals in secs
 DPU_MAX_TIMEOUT = 360
 DPU_TIME_INT = 30
-SYS_TIME_INT = 180
 
 # DPU Memory Threshold
 DPU_MEMORY_THRESHOLD = 90
@@ -52,33 +53,29 @@ def test_midplane_ip(duthosts, enum_rand_one_per_hwsku_hostname, platform_api_co
 def test_reboot_cause(duthosts, enum_rand_one_per_hwsku_hostname,
                       platform_api_conn, num_dpu_modules):    # noqa F811
     """
-    @summary: Verify `Reboot Cause`
+    @summary: Verify `Reboot Cause` using parallel execution.
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
 
-    for index in range(num_dpu_modules):
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis \
-                       module shutdown %s" % (dpu_name))["stdout_lines"]
+    dpu_names = [
+        module.get_name(platform_api_conn, index)
+        for index in range(num_dpu_modules)
+    ]
 
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_module_status,
-                                 duthost, "off",
-                                 dpu_name), "DPU is not operationally down")
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis modules startup %s" % (dpu_name))
+    logging.info("Shutting DOWN the DPUs in parallel")
+    dpus_shutdown_and_check(duthost, dpu_names, num_dpu_modules)
 
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_name),
-                      "DPU is not operationally up")
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_reboot_cause,
-                                 duthost,
-                                 dpu_name,
-                                 "Non-Hardware"), "Reboot cause is incorrect")
+    logging.info("Starting UP the DPUs in parallel")
+    dpus_startup_and_check(duthost, dpu_names, num_dpu_modules)
+
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Verify Reboot cause of all DPUs in parallel")
+        for dpu_name in dpu_names:
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_reboot_cause, duthost, dpu_name,
+                "Switch rebooted DPU"
+            )
 
 
 def test_pcie_link(duthosts, dpuhosts,
@@ -104,15 +101,8 @@ def test_pcie_link(duthosts, dpuhosts,
                   'PCIe Device Checking All Test ----------->>> PASSED',
                   "PCIe Link test failed'{}'".format(duthost.hostname))
 
-    for index in range(len(dpu_on_list)):
-        duthost.shell("sudo config chassis modules \
-                       shutdown %s" % (dpu_on_list[index]))
-
-    for index in range(len(dpu_on_list)):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "off", dpu_on_list[index]),
-                      "DPU is not operationally down")
+    logging.info("Shutting DOWN the DPUs in parallel")
+    dpus_shutdown_and_check(duthost, dpu_on_list, num_dpu_modules)
 
     output_pcie_info = duthost.command(CMD_PCIE_INFO)["stdout_lines"]
     pytest_assert(output_pcie_info[-1] ==
@@ -123,9 +113,9 @@ def test_pcie_link(duthosts, dpuhosts,
         duthost.shell("sudo config chassis modules \
                        startup %s" % (dpu_on_list[index]))
 
-    post_test_dpu_check(duthost, dpuhosts,
-                        dpu_on_list, dpu_off_list,
-                        ip_address_list)
+    post_test_dpus_check(duthost, dpuhosts,
+                         dpu_on_list, dpu_off_list,
+                         ip_address_list, num_dpu_modules)
 
     logging.info("Verifying output of '{}' on '{}'..."
                  .format(CMD_PCIE_INFO, duthost.hostname))
@@ -171,41 +161,25 @@ def test_restart_pmon(duthosts, enum_rand_one_per_hwsku_hostname,
 def test_system_health_state(duthosts, enum_rand_one_per_hwsku_hostname,
                              platform_api_conn, num_dpu_modules):  # noqa: F811
     """
-    @summary: To Verify `show system-health dpu` cli
+    @summary: To Verify `show system-health dpu` CLI
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
 
-    logging.info("Executing pre test check")
+    logging.info("Executing pre-test check")
     ip_address_list, dpu_on_list, dpu_off_list = pre_test_check(
-                                                 duthost,
-                                                 platform_api_conn,
-                                                 num_dpu_modules)
+        duthost, platform_api_conn, num_dpu_modules)
 
-    for index in range(len(dpu_on_list)):
-        logging.info("Shutting down {}".format(dpu_on_list[index]))
-        duthost.shell("sudo config chassis modules \
-                       shutdown %s" % (dpu_on_list[index]))
+    logging.info("Shutting DOWN the DPUs in parallel")
+    dpus_shutdown_and_check(duthost, dpu_on_list, num_dpu_modules)
 
     for index in range(len(dpu_on_list)):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "off", dpu_on_list[index]),
-                      "DPU is not operationally down")
-
         check_dpu_health_status(duthost, dpu_on_list[index],
                                 'Offline', 'down')
 
-    for index in range(len(dpu_on_list)):
-        logging.info("Powering up {}".format(dpu_on_list[index]))
-        duthost.shell("sudo config chassis modules \
-                       startup %s" % (dpu_on_list[index]))
+    logging.info("Starting UP the DPUs in parallel")
+    dpus_startup_and_check(duthost, dpu_on_list, num_dpu_modules)
 
     for index in range(len(dpu_on_list)):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, SYS_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up")
-
         check_dpu_health_status(duthost, dpu_on_list[index],
                                 'Online', 'up')
 
@@ -316,17 +290,25 @@ def test_system_health_summary(duthosts, dpuhosts,
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
 
+    logging.info("Collecting DPU informations")
+    ip_address_list, dpu_on_list, dpu_off_list = pre_test_check(
+                                                 duthost,
+                                                 platform_api_conn,
+                                                 num_dpu_modules)
+
+    logging.info("Checking DPU is completely UP")
+    post_test_dpus_check(duthost, dpuhosts,
+                         dpu_on_list, dpu_off_list,
+                         ip_address_list, num_dpu_modules)
+
     logging.info("Checking show system-health summary on Switch")
     output_health_summary = duthost.command("show system-health summary")
     result = parse_system_health_summary(output_health_summary['stdout'])
 
     pytest_assert(result, "Switch health status is not ok")
 
-    for index in range(num_dpu_modules):
+    for index in range(len(dpu_on_list)):
         dpu_name = module.get_name(platform_api_conn, index)
-        rc = check_dpu_module_status(duthost, "off", dpu_name)
-        if rc:
-            continue
 
         logging.info("Checking show system-health summary on {}"
                      .format(dpu_name))
diff --git a/tests/smartswitch/platform_tests/test_reload_dpu.py b/tests/smartswitch/platform_tests/test_reload_dpu.py
index 2dda1d6884b..0d24038c8cb 100644
--- a/tests/smartswitch/platform_tests/test_reload_dpu.py
+++ b/tests/smartswitch/platform_tests/test_reload_dpu.py
@@ -9,7 +9,7 @@
 from tests.common.reboot import reboot, REBOOT_TYPE_COLD
 from tests.smartswitch.common.device_utils_dpu import get_dpu_link_status,\
     check_dpu_ping_status, check_dpu_link_and_status, check_dpu_module_status,\
-    pre_test_check, post_test_switch_check, post_test_dpu_check,\
+    pre_test_check, post_test_switch_check, post_test_dpus_check,\
     check_dpu_reboot_cause, num_dpu_modules  # noqa: F401
 from tests.common.platform.device_utils import platform_api_conn, start_platform_api_service  # noqa: F401,F403
 
@@ -149,13 +149,13 @@ def test_dpu_status_post_dpu_kernel_panic(duthosts, dpuhosts,
     for index in range(len(dpu_on_list)):
         logging.info("Triggering Kernel Panic on %s" % (dpu_on_list[index]))
         dpu_on = dpu_on_list[index]
-        dpu_number = int(re.search(r'\d+', dpu_on).group())
-        dpuhosts[dpu_number].shell(kernel_panic_cmd, executable="/bin/bash")
+        dpu_id = int(re.search(r'\d+', dpu_on).group())
+        dpuhosts[dpu_id].shell(kernel_panic_cmd, executable="/bin/bash")
 
     logging.info("Executing post test dpu check")
-    post_test_dpu_check(duthost, dpuhosts,
-                        dpu_on_list, dpu_off_list,
-                        ip_address_list)
+    post_test_dpus_check(duthost, dpuhosts,
+                         dpu_on_list, dpu_off_list,
+                         ip_address_list, num_dpu_modules)
 
 
 def test_dpu_check_post_dpu_mem_exhaustion(duthosts, dpuhosts,
@@ -178,11 +178,10 @@ def test_dpu_check_post_dpu_mem_exhaustion(duthosts, dpuhosts,
                 "Triggering Memory Exhaustion on %s" % (dpu_on_list[index])
                 )
         dpu_on = dpu_on_list[index]
-        dpu_number = int(re.search(r'\d+', dpu_on).group())
-        dpuhosts[dpu_number].shell(memory_exhaustion_cmd,
-                                   executable="/bin/bash")
+        dpu_id = int(re.search(r'\d+', dpu_on).group())
+        dpuhosts[dpu_id].shell(memory_exhaustion_cmd, executable="/bin/bash")
 
     logging.info("Executing post test dpu check")
-    post_test_dpu_check(duthost, dpuhosts,
-                        dpu_on_list, dpu_off_list,
-                        ip_address_list)
+    post_test_dpus_check(duthost, dpuhosts,
+                         dpu_on_list, dpu_off_list,
+                         ip_address_list, num_dpu_modules)


Reviews:
user: login: vvolam
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:24:24Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:42:30Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:42:55Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:43:13Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T19:35:53Z
user: login: vvolam
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:22:32Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:55:24Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:55:48Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:55:59Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:56:06Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:56:12Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:56:25Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T20:58:02Z
user: login: JibinBao
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T10:04:36Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T19:27:31Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-29T01:45:26Z
user: login: JibinBao
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T04:02:41Z
user: login: vvolam
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T19:02:44Z
user: login: vvolam
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T21:41:40Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T22:44:20Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T22:44:30Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T22:44:43Z
user: login: nissampa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T23:03:57Z
user: login: vvolam
body: LGTM, Thanks!
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T01:23:39Z
user: login: bpar9
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-02T02:31:13Z

Review comments:
diff_hunk: @@ -352,7 +353,7 @@ def check_dpu_critical_processes(dpuhosts, dpu_number):
     If not, fails the case
     Args:
        dpuhosts: DPU Host handle
-       num_dpu_modules: Gets number of DPU modules
+       dpu_numer: Gets number of DPU modules
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: Can we instead say the variable name as "num_dpus"? dpu_number is not indicating that it is number of DPU modules. Thank you!
created_at: 2025-03-24T18:09:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -362,10 +363,13 @@ def check_dpu_critical_processes(dpuhosts, dpu_number):
 
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: Is dpu_number here the dpu index or the number of dpus?
created_at: 2025-03-24T18:10:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
+    pytest_assert(
+        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+                   check_dpu_module_status, duthost, "on", dpu_name),
+        f"DPU {dpu_name} is not operationally up post the operation"
+    )
+
+    dpu_number = int(re.search(r'\d+', dpu_name).group())
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: could you use "dpu_id" instead of dpu_number?
created_at: 2025-03-24T18:20:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,33 +53,83 @@ def test_midplane_ip(duthosts, enum_rand_one_per_hwsku_hostname, platform_api_co
 def test_reboot_cause(duthosts, enum_rand_one_per_hwsku_hostname,
                       platform_api_conn, num_dpu_modules):    # noqa F811
     """
-    @summary: Verify `Reboot Cause`
+    @summary: Verify `Reboot Cause` using parallel execution.
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
 
-    for index in range(num_dpu_modules):
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis \
-                       module shutdown %s" % (dpu_name))["stdout_lines"]
-
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_module_status,
-                                 duthost, "off",
-                                 dpu_name), "DPU is not operationally down")
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis modules startup %s" % (dpu_name))
-
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_name),
-                      "DPU is not operationally up")
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_reboot_cause,
-                                 duthost,
-                                 dpu_name,
-                                 "Non-Hardware"), "Reboot cause is incorrect")
+    dpu_names = [
+        module.get_name(platform_api_conn, index)
+        for index in range(num_dpu_modules)
+    ]
+
+    # Shutdown all DPUs in parallel
path: tests/smartswitch/platform_tests/test_platform_dpu.py
user: login: vvolam
body: Could you define a function for shutdown and wait()? And same for start-up and wait. So it can be called in multiple tests directly?
created_at: 2025-03-24T18:24:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -352,7 +353,7 @@ def check_dpu_critical_processes(dpuhosts, dpu_number):
     If not, fails the case
     Args:
        dpuhosts: DPU Host handle
-       num_dpu_modules: Gets number of DPU modules
+       dpu_numer: Gets number of DPU modules
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Makes sense. Changed it
created_at: 2025-03-24T18:42:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -362,10 +363,13 @@ def check_dpu_critical_processes(dpuhosts, dpu_number):
 
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Wrong description. Changed it.  
created_at: 2025-03-24T18:42:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
+    pytest_assert(
+        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+                   check_dpu_module_status, duthost, "on", dpu_name),
+        f"DPU {dpu_name} is not operationally up post the operation"
+    )
+
+    dpu_number = int(re.search(r'\d+', dpu_name).group())
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: changed it to dpu_id
created_at: 2025-03-24T18:43:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,33 +53,83 @@ def test_midplane_ip(duthosts, enum_rand_one_per_hwsku_hostname, platform_api_co
 def test_reboot_cause(duthosts, enum_rand_one_per_hwsku_hostname,
                       platform_api_conn, num_dpu_modules):    # noqa F811
     """
-    @summary: Verify `Reboot Cause`
+    @summary: Verify `Reboot Cause` using parallel execution.
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
 
-    for index in range(num_dpu_modules):
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis \
-                       module shutdown %s" % (dpu_name))["stdout_lines"]
-
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_module_status,
-                                 duthost, "off",
-                                 dpu_name), "DPU is not operationally down")
-        dpu_name = module.get_name(platform_api_conn, index)
-        duthost.shell("sudo config chassis modules startup %s" % (dpu_name))
-
-    for index in range(num_dpu_modules):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_name),
-                      "DPU is not operationally up")
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                                 check_dpu_reboot_cause,
-                                 duthost,
-                                 dpu_name,
-                                 "Non-Hardware"), "Reboot cause is incorrect")
+    dpu_names = [
+        module.get_name(platform_api_conn, index)
+        for index in range(num_dpu_modules)
+    ]
+
+    # Shutdown all DPUs in parallel
path: tests/smartswitch/platform_tests/test_platform_dpu.py
user: login: nissampa
body: Added it. 
created_at: 2025-03-24T19:35:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: could you rename this test to post_test_dpu_check(), as this test is checking for single DPU??
created_at: 2025-03-24T20:14:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: could you use 'UP' instead of up? this clearly tells the state we are checking. Please update all other log messages as well to explicitly mention 'UP' or 'DOWN' or 'ON' or 'OFF' to be clear.
created_at: 2025-03-24T20:15:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
+    pytest_assert(
+        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+                   check_dpu_module_status, duthost, "on", dpu_name),
+        f"DPU {dpu_name} is not operationally up post the operation"
+    )
+
+    dpu_id = int(re.search(r'\d+', dpu_name).group())
+    logging.info(f"Checking critical processes on {dpu_name}")
+    pytest_assert(
+        wait_until(
+            DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+            check_dpu_critical_processes, dpuhosts, dpu_id),
+        f"Crictical process check for {dpu_name} has been failed"
+    )
+
+    logging.info(f"Checking reboot cause of {dpu_name}")
+    pytest_assert(
+        wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
+                   check_dpu_reboot_cause, duthost, dpu_name, "Non-Hardware"),
+        f"Reboot cause for DPU {dpu_name} is incorrect"
+    )
+
+
 def post_test_dpu_check(duthost, dpuhosts,
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: Rename to post_test_dpus_check()
created_at: 2025-03-24T20:17:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: Remove this empty line?
created_at: 2025-03-24T20:19:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
+
+        # Verify all DPUs are down in parallel
+        futures_down = {
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "off", dpu_name
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_down):
+            pytest_assert(
+                future.result(),
+                f"DPU {futures_down[future]} is not operationally down"
+            )
+
+
+def dpu_startup_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU startup for given DPU list
+    Waits and checks parallely whether DPU is actually up
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be startup
+    Returns:
+       Returns Nothing
+    """
+
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: Remove this empty line?
created_at: 2025-03-24T20:19:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: could you use name as dpus instead of dpu for this and next function?
created_at: 2025-03-24T20:20:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -104,15 +107,8 @@ def test_pcie_link(duthosts, dpuhosts,
                   'PCIe Device Checking All Test ----------->>> PASSED',
                   "PCIe Link test failed'{}'".format(duthost.hostname))
 
-    for index in range(len(dpu_on_list)):
-        duthost.shell("sudo config chassis modules \
-                       shutdown %s" % (dpu_on_list[index]))
-
-    for index in range(len(dpu_on_list)):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "off", dpu_on_list[index]),
-                      "DPU is not operationally down")
+    logging.info("Shutting down the dpus in parallel")
+    dpu_shutdown_and_check(duthost, dpu_on_list)
 
     output_pcie_info = duthost.command(CMD_PCIE_INFO)["stdout_lines"]
     pytest_assert(output_pcie_info[-1] ==
path: tests/smartswitch/platform_tests/test_platform_dpu.py
user: login: vvolam
body: can we use dpu_startup_and_check() function at line 119??
created_at: 2025-03-24T20:22:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it. 
created_at: 2025-03-24T20:55:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it in respectively places
created_at: 2025-03-24T20:55:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -429,6 +433,42 @@ def post_test_switch_check(duthost, localhost,
     return
 
 
+def post_run_checks(duthost, dpuhosts, dpu_name):
+    """
+    Runs all required checks for a given DPU
+    Args:
+       duthost: Host handle
+       dpuhosts: DPU Host handle
+       dpu_name: Name of the DPU
+    Returns:
+       Returns Nothing
+
+    """
+
+    logging.info(f"Checking {dpu_name} is up post test")
+    pytest_assert(
+        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+                   check_dpu_module_status, duthost, "on", dpu_name),
+        f"DPU {dpu_name} is not operationally up post the operation"
+    )
+
+    dpu_id = int(re.search(r'\d+', dpu_name).group())
+    logging.info(f"Checking critical processes on {dpu_name}")
+    pytest_assert(
+        wait_until(
+            DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
+            check_dpu_critical_processes, dpuhosts, dpu_id),
+        f"Crictical process check for {dpu_name} has been failed"
+    )
+
+    logging.info(f"Checking reboot cause of {dpu_name}")
+    pytest_assert(
+        wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
+                   check_dpu_reboot_cause, duthost, dpu_name, "Non-Hardware"),
+        f"Reboot cause for DPU {dpu_name} is incorrect"
+    )
+
+
 def post_test_dpu_check(duthost, dpuhosts,
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Renamed it.
created_at: 2025-03-24T20:55:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Removed it.
created_at: 2025-03-24T20:56:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
+
+        # Verify all DPUs are down in parallel
+        futures_down = {
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "off", dpu_name
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_down):
+            pytest_assert(
+                future.result(),
+                f"DPU {futures_down[future]} is not operationally down"
+            )
+
+
+def dpu_startup_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU startup for given DPU list
+    Waits and checks parallely whether DPU is actually up
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be startup
+    Returns:
+       Returns Nothing
+    """
+
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Removed it.
created_at: 2025-03-24T20:56:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,89 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_run_checks, duthost, dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpu_shutdown_and_check(duthost, dpu_list):
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed the name for this and next function.
created_at: 2025-03-24T20:56:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -104,15 +107,8 @@ def test_pcie_link(duthosts, dpuhosts,
                   'PCIe Device Checking All Test ----------->>> PASSED',
                   "PCIe Link test failed'{}'".format(duthost.hostname))
 
-    for index in range(len(dpu_on_list)):
-        duthost.shell("sudo config chassis modules \
-                       shutdown %s" % (dpu_on_list[index]))
-
-    for index in range(len(dpu_on_list)):
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "off", dpu_on_list[index]),
-                      "DPU is not operationally down")
+    logging.info("Shutting down the dpus in parallel")
+    dpu_shutdown_and_check(duthost, dpu_on_list)
 
     output_pcie_info = duthost.command(CMD_PCIE_INFO)["stdout_lines"]
     pytest_assert(output_pcie_info[-1] ==
path: tests/smartswitch/platform_tests/test_platform_dpu.py
user: login: nissampa
body: It will be redundant I feel. 
We discussed to include post_test_dpus_check here for covering even critical process after we bring up the pcie link. So, post_test_dpus_check function basically covers what dpu_startup_and_check does it as well. 
created_at: 2025-03-24T20:58:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,88 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
path: tests/smartswitch/common/device_utils_dpu.py
user: login: JibinBao
body: Can we check the results? Otherwise, when it returns an error, we will not catch it.
created_at: 2025-03-25T10:03:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,88 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
+
+        # Verify all DPUs are down in parallel
+        futures_down = {
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "off", dpu_name
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_down):
+            pytest_assert(
+                future.result(),
+                f"DPU {futures_down[future]} is not operationally DOWN"
+            )
+
+
+def dpus_startup_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU startup for given DPU list
+    Waits and checks parallely whether DPU is actually UP
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be startup
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Power up all DPUs in parallel
+        futures_startup = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules startup {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_startup):
+            future.result()  # Ensure execution completes
path: tests/smartswitch/common/device_utils_dpu.py
user: login: JibinBao
body: Can we check the results? Otherwise, when it returns an error, we will not catch it.
created_at: 2025-03-25T10:03:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,88 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: added error check
created_at: 2025-03-25T19:27:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,88 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Shutdown all DPUs in parallel
+        futures_shutdown = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules shutdown {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_shutdown):
+            future.result()  # Ensure execution completes
+
+        # Verify all DPUs are down in parallel
+        futures_down = {
+            executor.submit(
+                wait_until, DPU_MAX_TIMEOUT, DPU_TIME_INT, 0,
+                check_dpu_module_status, duthost, "off", dpu_name
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_down):
+            pytest_assert(
+                future.result(),
+                f"DPU {futures_down[future]} is not operationally DOWN"
+            )
+
+
+def dpus_startup_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU startup for given DPU list
+    Waits and checks parallely whether DPU is actually UP
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be startup
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Power up all DPUs in parallel
+        futures_startup = {
+            executor.submit(
+                duthost.shell,
+                f"sudo config chassis modules startup {dpu_name}"
+            ): dpu_name for dpu_name in dpu_list
+        }
+        for future in concurrent.futures.as_completed(futures_startup):
+            future.result()  # Ensure execution completes
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: added error check.
created_at: 2025-03-29T01:45:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -345,27 +346,30 @@ def check_dpu_health_status(duthost, dpu_name,
     return
 
 
-def check_dpu_critical_processes(dpuhosts, dpu_number):
+def check_dpu_critical_processes(dpuhosts, dpu_id):
 
     """
-    Checks all critical processes are up on DPU
+    Checks all critical processes are UP on DPU
     If not, fails the case
     Args:
        dpuhosts: DPU Host handle
-       num_dpu_modules: Gets number of DPU modules
+       dpu_id: Gets DPU ID
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: I think here we are passing DPU ID, right? could you fix summary?
created_at: 2025-04-01T18:46:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,97 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: could you use SafeThreadPoolExecutor instead https://github.com/sonic-net/sonic-mgmt/blob/ee2e0d4076a177c11d79ff214d34b78bad8dbe83/tests/common/helpers/multi_thread_utils.py#L6
created_at: 2025-04-01T19:00:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,97 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                future.result()
+            except Exception as e:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: With SafeThreadPoolExecutor we don't have to print Exception.
created_at: 2025-04-01T19:02:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +485,90 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Post test DPUs check in parallel")
+        for dpu in dpu_on_list:
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu)
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: vvolam
body: @nissampa You could use same SafeThreadPoolExecutor for all these concurrent executions.
created_at: 2025-04-01T21:41:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -345,27 +346,30 @@ def check_dpu_health_status(duthost, dpu_name,
     return
 
 
-def check_dpu_critical_processes(dpuhosts, dpu_number):
+def check_dpu_critical_processes(dpuhosts, dpu_id):
 
     """
-    Checks all critical processes are up on DPU
+    Checks all critical processes are UP on DPU
     If not, fails the case
     Args:
        dpuhosts: DPU Host handle
-       num_dpu_modules: Gets number of DPU modules
+       dpu_id: Gets DPU ID
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it.
created_at: 2025-04-01T22:44:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,97 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu): dpu
+            for dpu in dpu_on_list
+        }
 
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                future.result()
+            except Exception as e:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it.
created_at: 2025-04-01T22:44:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +484,97 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it. 
created_at: 2025-04-01T22:44:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,31 +485,90 @@ def post_test_dpu_check(duthost, dpuhosts,
        Returns Nothing
     """
 
-    for index in range(len(dpu_on_list)):
-        logging.info(
-            "Checking %s is up post test" % (dpu_on_list[index])
-            )
-
-        pytest_assert(wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                      check_dpu_module_status,
-                      duthost, "on", dpu_on_list[index]),
-                      "DPU is not operationally up post the operation")
-
-        dpu_number = int(re.search(r'\d+', dpu_on_list[index]).group())
-        logging.info("Checking crictical processes \
-                      on %s" % (dpu_on_list[index]))
-        wait_until(DPU_MAX_TIMEOUT, DPU_MAX_TIME_INT, 0,
-                   check_dpu_critical_processes,
-                   dpuhosts, dpu_number)
-
-        logging.info("Checking reboot cause of %s" % (dpu_on_list[index]))
-        pytest_assert(wait_until(REBOOT_CAUSE_TIMEOUT, REBOOT_CAUSE_INT, 0,
-                      check_dpu_reboot_cause,
-                      duthost, dpu_on_list[index], "Non-Hardware"),
-                      "Reboot cause is not correct")
+    with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor:
+        logging.info("Post test DPUs check in parallel")
+        for dpu in dpu_on_list:
+            executor.submit(post_test_dpu_check, duthost,
+                            dpuhosts, dpu)
 
     logging.info("Checking all powered on DPUs connectivity")
     ping_status = check_dpu_ping_status(duthost, ip_address_list)
     pytest_assert(ping_status == 1, "Ping to one or more DPUs has failed")
 
     return
+
+
+def dpus_shutdown_and_check(duthost, dpu_list):
+    """
+    Parallely Execute DPU shutdown for given DPU list
+    Waits and checks parallely whether DPU is actually down
+    Args:
+       duthost: Host handle
+       dpu_list: List of DPUs to be shutdown
+    Returns:
+       Returns Nothing
+    """
+    with concurrent.futures.ThreadPoolExecutor() as executor:
path: tests/smartswitch/common/device_utils_dpu.py
user: login: nissampa
body: Changed it. 
created_at: 2025-04-01T23:03:57Z
author_association: CONTRIBUTOR

PR comments:
user: login: mssonicbld
created_at: 2025-03-19T22:45:38Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-19T22:45:48Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-19T23:11:01Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-19T23:11:13Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-20T21:56:24Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-20T21:56:34Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-20T22:04:22Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-20T22:04:34Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: nissampa
created_at: 2025-03-20T22:05:37Z
body: @vvolam, @JibinBao 

Please review
Kindly add Gagan as well 
author_association: CONTRIBUTOR
user: login: mssonicbld
created_at: 2025-03-24T19:35:43Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T19:35:54Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T19:36:12Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T19:36:24Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T19:38:49Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T19:38:59Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T19:53:29Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T19:53:41Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T19:57:52Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T19:58:05Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T20:55:18Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T20:55:30Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-24T21:00:17Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-24T21:00:33Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-25T19:29:43Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-25T19:29:54Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-25T19:32:08Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-25T19:32:20Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: vvolam
created_at: 2025-03-29T00:24:14Z
body: @nissampa Could you please update PR description with details and also the platform where it is tested?
author_association: CONTRIBUTOR
user: login: mssonicbld
created_at: 2025-03-29T01:46:30Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-29T01:46:41Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-29T01:46:46Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-29T01:46:58Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: nissampa
created_at: 2025-04-01T00:03:01Z
body: @JibinBao & @vvolam 

Can you help me in approving it if things are good with this PR ?
author_association: CONTRIBUTOR
user: login: mssonicbld
created_at: 2025-04-01T20:51:47Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-01T20:52:00Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-01T20:52:08Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-01T20:52:19Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-01T21:17:03Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-01T21:17:14Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-01T23:03:48Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-01T23:04:00Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE