Repository: cdent/gabbi

Issue:
exceptions that happen in start_fixture() are swallowed during the test discovery process
The discovery will fail and end up producing a seemingly non-related error like `math domain error` and the real exception is nowhere to be found.

This needs to be narrowed to a minimal test case and then figured out.

What seems likely is that something about the test suites is being processed is not following unittest/testtools rules of failure.

Discussion:
cdent: I'm struggling (because of other mental commitments and general stupidness) to come up with a reasonable way to add tests for this. If anyone has ideas please feel free to let me know.

Patch:
diff --git a/gabbi/suite.py b/gabbi/suite.py
index 765a929..f6d3868 100644
--- a/gabbi/suite.py
+++ b/gabbi/suite.py
@@ -16,6 +16,7 @@ This suite has two features: the contained tests are ordered and there
 are suite-level fixtures that operate as context managers.
 """
 
+import sys
 import unittest
 
 from wsgi_intercept import interceptor
@@ -58,6 +59,26 @@ class GabbiSuite(unittest.TestSuite):
         except unittest.SkipTest as exc:
             for test in self._tests:
                 result.addSkip(test, str(exc))
+        # If we have an exception in the nested fixtures, that means
+        # there's been an exception somewhere in the cycle other
+        # than a specific test (as that would have been caught
+        # already), thus from a fixture. If that exception were to
+        # continue to raise here, then some test runners would
+        # swallow it and the traceback of the failure would be
+        # undiscoverable. To ensure the traceback is reported (via
+        # the testrunner) to a human, the first test in the suite is
+        # marked as having an error (it's fixture failed) and then
+        # the entire suite is skipped, and the result stream told
+        # we're done. If there are no tests (an empty suite) the
+        # exception is re-raised.
+        except Exception as exc:
+            if self._tests:
+                result.addError(self._tests[0], sys.exc_info())
+                for test in self._tests:
+                    result.addSkip(test, 'fixture failure')
+                result.stop()
+            else:
+                raise
 
         return result
 


Test patch:
diff --git a/gabbi/tests/test_suite.py b/gabbi/tests/test_suite.py
new file mode 100644
index 0000000..eb1bd22
--- /dev/null
+++ b/gabbi/tests/test_suite.py
@@ -0,0 +1,54 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Unit tests for the gabbi.suite.
+"""
+
+import sys
+import unittest
+
+from gabbi import fixture
+from gabbi import suitemaker
+
+VALUE_ERROR = 'value error sentinel'
+
+
+class FakeFixture(fixture.GabbiFixture):
+
+    def start_fixture(self):
+        raise ValueError(VALUE_ERROR)
+
+
+class SuiteTest(unittest.TestCase):
+
+    def test_suite_catches_fixture_fail(self):
+        """When a fixture fails in start_fixture it should fail
+        the first test in the suite and skip the others.
+        """
+        loader = unittest.defaultTestLoader
+        result = unittest.TestResult()
+        test_data = {'fixtures': ['FakeFixture'],
+                     'tests': [{'name': 'alpha', 'GET': '/'},
+                               {'name': 'beta', 'GET': '/'}]}
+        test_suite = suitemaker.test_suite_from_dict(
+            loader, 'foo', test_data, '.', 'localhost',
+            80, sys.modules[__name__], None)
+
+        test_suite.run(result)
+
+        self.assertEqual(2, len(result.skipped))
+        self.assertEqual(1, len(result.errors))
+
+        errored_test, trace = result.errors[0]
+
+        self.assertIn('foo_alpha', str(errored_test))
+        self.assertIn(VALUE_ERROR, trace)