Repository: microsoft/superbenchmark

Issue:
'sb deploy' is expected to exit with non-zero when failed
## Summary

This issue was found using v0.6.0 release. In the system, the ansible was not setup properly because of a test environment issue. When  I ran 'sb deploy -f local.ini -i superbench/superbench:v0.6.0-cuda11.1.1'. It has error message like below. Although the error message said ansible.py return code 127, the sb program exit with 0.

[2022-09-09 18:49:10,573 N000000:30359][runner.py:43][INFO] Runner writes to: /home/aiscadmin/superbench/outputs/2022-09-09_18-49-10.
[2022-09-09 18:49:10,622 N000000:30359][runner.py:48][INFO] Runner will run: ['gpu-burn', 'nccl-bw:default', 'nccl-bw:gdr-only', 'ib-loopback', 'mem-bw', 'gpu-copy-bw:correctness', 'gpu-copy-bw:perf', 'kernel-launch', 'gemm-flops', 'cudnn-function', 'cublas-function', 'matmul', 'sharding-matmul', 'computation-communication-overlap', 'ort-inference', 'tensorrt-inference', 'gpt_models', 'bert_models', 'lstm_models', 'resnet_models', 'densenet_models', 'vgg_models']
[2022-09-09 18:49:10,622 N000000:30359][runner.py:165][INFO] Preparing SuperBench environment.
[2022-09-09 18:49:10,622 N000000:30359][ansible.py:125][INFO] Run playbook deploy.yaml ...
The command was not found or was not executable: ansible-playbook.
[2022-09-09 18:49:10,628 N000000:30359][ansible.py:80][WARNING] Run failed, return code 127.

$ echo $?
0

## How to repro

Setup superbench normally. Before running 'sb deploy', remove the ~/ .ansible directory. Then run 'sb deploy' like above.






Discussion:

Resolving patch:
diff --git a/superbench/cli/_handler.py b/superbench/cli/_handler.py
--- a/superbench/cli/_handler.py
+++ b/superbench/cli/_handler.py
@@ -3,6 +3,7 @@
 
 """SuperBench CLI command handler."""
 
+import sys
 from pathlib import Path
 from importlib_metadata import version, PackageNotFoundError
 
@@ -252,6 +253,8 @@ def deploy_command_handler(
 
     runner = SuperBenchRunner(sb_config, docker_config, ansible_config, sb_output_dir)
     runner.deploy()
+    if runner.get_failure_count() != 0:
+        sys.exit(runner.get_failure_count())
 
 
 def run_command_handler(
@@ -307,3 +310,5 @@ def run_command_handler(
 
     runner = SuperBenchRunner(sb_config, docker_config, ansible_config, sb_output_dir)
     runner.run()
+    if runner.get_failure_count() != 0:
+        sys.exit(runner.get_failure_count())
diff --git a/superbench/runner/ansible.py b/superbench/runner/ansible.py
--- a/superbench/runner/ansible.py
+++ b/superbench/runner/ansible.py
@@ -27,6 +27,7 @@ def __init__(self, config):
             'cmdline': '--forks 128',
         }
         self._head_host = None
+        self.failure_count = 0
         if config:
             inventory_file = getattr(config, 'host_file', None)
             inventory_list = getattr(config, 'host_list', None)
@@ -77,6 +78,7 @@ def run(self, ansible_config, sudo=False):    # pragma: no cover
         if r.rc == 0:
             logger.info('Run succeed, return code {}.'.format(r.rc))
         else:
+            self.failure_count += 1
             logger.warning('Run failed, return code {}.'.format(r.rc))
         return r.rc
 
diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -160,6 +160,14 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
             logger.warning('Unknown mode %s.', mode.name)
         return mode_command.strip()
 
+    def get_failure_count(self):
+        """Get failure count during Ansible run.
+
+        Return:
+            int: Failure count.
+        """
+        return self._ansible_client.failure_count
+
     def deploy(self):    # pragma: no cover
         """Deploy SuperBench environment."""
         logger.info('Preparing SuperBench environment.')


Test patch:
diff --git a/tests/cli/test_sb.py b/tests/cli/test_sb.py
--- a/tests/cli/test_sb.py
+++ b/tests/cli/test_sb.py
@@ -8,6 +8,7 @@
 from functools import wraps
 from knack.testsdk import ScenarioTest, StringContainCheck, NoneCheck, JMESPathCheck
 from pathlib import Path
+from unittest import mock
 
 import superbench
 from superbench.cli import SuperBenchCLI
@@ -53,8 +54,10 @@ def test_sb_version(self):
         """Test sb version."""
         self.cmd('sb version', checks=[StringContainCheck(superbench.__version__)])
 
-    def test_sb_deploy(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_deploy(self, mocked_failure_count):
         """Test sb deploy."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb deploy --host-list localhost', checks=[NoneCheck()])
 
     def test_sb_deploy_no_host(self):
@@ -65,12 +68,16 @@ def test_sb_exec(self):
         """Test sb exec."""
         self.cmd('sb exec --config-override superbench.enable=["none"]', checks=[NoneCheck()])
 
-    def test_sb_run(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_run(self, mocked_failure_count):
         """Test sb run."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb run --host-list localhost --config-override superbench.enable=none', checks=[NoneCheck()])
 
-    def test_sb_run_skipdocker(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_run_skipdocker(self, mocked_failure_count):
         """Test sb run without docker."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb run -l localhost -C superbench.enable=none --no-docker', checks=[NoneCheck()])
 
     def test_sb_run_no_docker_auth(self):
diff --git a/tests/runner/test_runner.py b/tests/runner/test_runner.py
--- a/tests/runner/test_runner.py
+++ b/tests/runner/test_runner.py
@@ -41,6 +41,10 @@ def test_set_logger(self):
         expected_log_file = Path(self.runner._sb_output_dir) / 'sb-run.log'
         self.assertTrue(expected_log_file.is_file())
 
+    def test_get_failure_count(self):
+        """Test get_failure_count."""
+        self.assertEqual(0, self.runner.get_failure_count())
+
     def test_get_mode_command(self):
         """Test __get_mode_command."""
         test_cases = [