Repository: google/mobly

Issue:
Exceptions in `setup_test` should leave the test in `ERROR` status
Regardless of the type of the exception, `setup_test` error should cause `ERROR` status.

This is different from a test method.
In a test method, an exception based on signals.TestFailure should cause the test to exit with `FAILED` status.

This is to be consistent with pyunit's behavior.


Discussion:
setup_class as well

Resolving patch:
diff --git a/mobly/signals.py b/mobly/signals.py
--- a/mobly/signals.py
+++ b/mobly/signals.py
@@ -46,6 +46,10 @@ def __str__(self):
         return 'Details=%s, Extras=%s' % (self.details, self.extras)
 
 
+class TestError(TestSignal):
+    """Raised when a test has an unexpected error."""
+
+
 class TestFailure(TestSignal):
     """Raised when a test has failed."""
 


Test patch:
diff --git a/mobly/base_test.py b/mobly/base_test.py
--- a/mobly/base_test.py
+++ b/mobly/base_test.py
@@ -17,6 +17,7 @@
 import functools
 import inspect
 import logging
+import sys
 
 from mobly import logger
 from mobly import records
@@ -316,7 +317,7 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
 
         Executes setup_test, the test method, and teardown_test; then creates a
         records.TestResultRecord object with the execution information and adds
-        the record to the test class's test results.
+        the record to the test class's test result s.
 
         Args:
             test_name: Name of the test.
@@ -330,7 +331,12 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
         teardown_test_failed = False
         try:
             try:
-                self._setup_test(test_name)
+                try:
+                    self._setup_test(test_name)
+                except signals.TestFailure as e:
+                    new_e = signals.TestError(e.details, e.extras)
+                    _, _, new_e.__traceback__ = sys.exc_info()
+                    raise new_e
                 if args or kwargs:
                     test_method(*args, **kwargs)
                 else:
diff --git a/tests/mobly/base_test_test.py b/tests/mobly/base_test_test.py
--- a/tests/mobly/base_test_test.py
+++ b/tests/mobly/base_test_test.py
@@ -239,11 +239,13 @@ def test_something(self):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run(test_names=["test_something"])
-        actual_record = bt_cls.results.failed[0]
+        actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
+        # Make sure the full stacktrace of `setup_test` is preserved.
+        self.assertTrue('self.setup_test()' in actual_record.stacktrace)
         self.assertIsNone(actual_record.extras)
-        expected_summary = ("Error 0, Executed 1, Failed 1, Passed 0, "
+        expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
@@ -407,6 +409,7 @@ def test_something(self):
 
     def test_procedure_function_gets_correct_record(self):
         on_fail_mock = mock.MagicMock()
+
         class MockBaseTest(base_test.BaseTestClass):
             def on_fail(self, record):
                 on_fail_mock.record = record
@@ -418,12 +421,16 @@ def test_something(self):
         bt_cls.run()
         actual_record = bt_cls.results.failed[0]
         self.assertEqual(actual_record.test_name, 'test_something')
-        self.assertEqual(on_fail_mock.record.test_name, actual_record.test_name)
-        self.assertEqual(on_fail_mock.record.begin_time, actual_record.begin_time)
+        self.assertEqual(on_fail_mock.record.test_name,
+                         actual_record.test_name)
+        self.assertEqual(on_fail_mock.record.begin_time,
+                         actual_record.begin_time)
         self.assertEqual(on_fail_mock.record.end_time, actual_record.end_time)
-        self.assertEqual(on_fail_mock.record.stacktrace, actual_record.stacktrace)
+        self.assertEqual(on_fail_mock.record.stacktrace,
+                         actual_record.stacktrace)
         self.assertEqual(on_fail_mock.record.extras, actual_record.extras)
-        self.assertEqual(on_fail_mock.record.extra_errors, actual_record.extra_errors)
+        self.assertEqual(on_fail_mock.record.extra_errors,
+                         actual_record.extra_errors)
         # But they are not the same object.
         self.assertIsNot(on_fail_mock.record, actual_record)
 
@@ -989,6 +996,23 @@ def test_func(self):
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertEqual(actual_record.extras, MOCK_EXTRA)
 
+    def test_skip_in_setup_test(self):
+        class MockBaseTest(base_test.BaseTestClass):
+            def setup_test(self):
+                asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA)
+
+            def test_func(self):
+                never_call()
+
+        bt_cls = MockBaseTest(self.mock_test_cls_configs)
+        bt_cls.run(test_names=["test_func"])
+        actual_record = bt_cls.results.skipped[0]
+        self.assertIsNotNone(actual_record.begin_time)
+        self.assertIsNotNone(actual_record.end_time)
+        self.assertEqual(actual_record.test_name, "test_func")
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
+        self.assertEqual(actual_record.extras, MOCK_EXTRA)
+
     def test_unpack_userparams_required(self):
         """Missing a required param should raise an error."""
         required = ["some_param"]