Repository: sjagoe/usagi

Issue:
Allow setting maxDiff at the testCase level and test level
Currently if a body assertion fails there is no way to see the whole diff to resolve the issue.

We need a mechanism to set the underlying TestCase `maxDiff` attribute on a test-case level and an individual test level.

Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index ef6028e..27fa8ce 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -10,6 +10,8 @@ Enhancements
 
 * Added an option to the body assertion to filter json responses with a
   ``jq`` (https://pypi.python.org/pypi/jq) filter (#44).
+* Added an option to cases and tests to allow setting the
+  unittest.TestCase.maxDiff parameter (#47).
 
 
 Version 0.2.0
diff --git a/usagi/schema.py b/usagi/schema.py
index 20915a9..5b0f11a 100644
--- a/usagi/schema.py
+++ b/usagi/schema.py
@@ -118,12 +118,18 @@ SCHEMA = {
                     },
                     'minItems': 1,
                 },
+                'max-diff': {
+                    '$ref': '#/definitions/max-diff',
+                },
             },
             'required': ['name', 'tests'],
         },
         'test': {
             'type': 'object',
             'properties': {
+                'max-diff': {
+                    '$ref': '#/definitions/max-diff',
+                },
                 'parameters': {'type': 'object'},
                 'url': {
                     'oneOf': [
@@ -155,5 +161,9 @@ SCHEMA = {
             },
             'required': ['url', 'name'],
         },
+        'max-diff': {
+            'type': ['number', 'null'],
+            'description': 'Set the case maxDiff option to control error output',  # noqa
+        },
     },
 }


Test patch:
diff --git a/usagi/tests/test_schema.py b/usagi/tests/test_schema.py
index 21fac64..f270e32 100644
--- a/usagi/tests/test_schema.py
+++ b/usagi/tests/test_schema.py
@@ -337,3 +337,137 @@ class TestSchema(unittest.TestCase):
         # Validation fails
         with self.assertRaises(ValidationError):
             jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_case_max_diff_null(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              max-diff: null
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation succeeds
+        jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_case_max_diff_number(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              max-diff: 12345
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation succeeds
+        jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_case_max_diff_string(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              max-diff: '12345'
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation fails
+        with self.assertRaises(ValidationError):
+            jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_test_max_diff_null(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+                  max-diff: null
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation succeeds
+        jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_test_max_diff_number(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+                  max-diff: 12345
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation succeeds
+        jsonschema.validate(test_data, SCHEMA)
+
+    def test_schema_test_max_diff_string(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              tests:
+                - name: "Another URL"
+                  url: "/another"
+                  max-diff: '12345'
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # Validation fails
+        with self.assertRaises(ValidationError):
+            jsonschema.validate(test_data, SCHEMA)
diff --git a/usagi/tests/test_web_test.py b/usagi/tests/test_web_test.py
index 266f86c..4a72983 100644
--- a/usagi/tests/test_web_test.py
+++ b/usagi/tests/test_web_test.py
@@ -25,7 +25,7 @@ from ..plugins.test_parameters import (
 )
 from ..config import Config
 from ..utils import create_session
-from ..web_test import WebPoll, WebTest
+from ..web_test import WebPoll, WebTest, _Default
 
 
 class TestWebTest(unittest.TestCase):
@@ -72,6 +72,77 @@ class TestWebTest(unittest.TestCase):
         # Then
         self.assertIs(test.session, session)
         self.assertIs(test.config, config)
+        self.assertIs(test.max_diff, _Default)
+        self.assertEqual(test.name, name)
+        self.assertEqual(test.url, expected_url)
+        self.assertEqual(self._get_web_test_method(test), 'GET')
+        self.assertEqual(len(test.assertions), 0)
+
+    def test_from_dict_null_max_diff(self):
+        # Given
+        config = Config.from_dict({'host': 'test.invalid'}, __file__)
+        session = create_session()
+        name = 'A test'
+        url = '/api/test'
+        test_spec = {
+            'name': name,
+            'url': url,
+            'max-diff': None,
+        }
+        expected_url = urllib.parse.urlunparse(
+            urllib.parse.ParseResult(
+                config.scheme,
+                config.host,
+                url,
+                None,
+                None,
+                None,
+            ),
+        )
+
+        # When
+        test = WebTest.from_dict(
+            session, test_spec, config, {}, self.test_parameter_plugins)
+
+        # Then
+        self.assertIs(test.session, session)
+        self.assertIs(test.config, config)
+        self.assertIsNone(test.max_diff)
+        self.assertEqual(test.name, name)
+        self.assertEqual(test.url, expected_url)
+        self.assertEqual(self._get_web_test_method(test), 'GET')
+        self.assertEqual(len(test.assertions), 0)
+
+    def test_from_dict_integer_max_diff(self):
+        # Given
+        config = Config.from_dict({'host': 'test.invalid'}, __file__)
+        session = create_session()
+        name = 'A test'
+        url = '/api/test'
+        test_spec = {
+            'name': name,
+            'url': url,
+            'max-diff': 5001,
+        }
+        expected_url = urllib.parse.urlunparse(
+            urllib.parse.ParseResult(
+                config.scheme,
+                config.host,
+                url,
+                None,
+                None,
+                None,
+            ),
+        )
+
+        # When
+        test = WebTest.from_dict(
+            session, test_spec, config, {}, self.test_parameter_plugins)
+
+        # Then
+        self.assertIs(test.session, session)
+        self.assertIs(test.config, config)
+        self.assertEqual(test.max_diff, 5001)
         self.assertEqual(test.name, name)
         self.assertEqual(test.url, expected_url)
         self.assertEqual(self._get_web_test_method(test), 'GET')
@@ -106,6 +177,7 @@ class TestWebTest(unittest.TestCase):
         # Then
         self.assertIs(test.session, session)
         self.assertIs(test.config, config)
+        self.assertIs(test.max_diff, _Default)
         self.assertEqual(test.name, name)
         self.assertEqual(test.url, expected_url)
         self.assertEqual(self._get_web_test_method(test), 'POST')
@@ -208,6 +280,73 @@ class TestWebTest(unittest.TestCase):
         self.assertEqual(args, (204, 200))
         self.assertIn('msg', kwargs)
 
+        # No maxDiff has been set
+        self.assertIsInstance(case.maxDiff, Mock)
+
+    @responses.activate
+    def test_run_null_max_diff(self):
+        # Given
+        config = Config.from_dict({'host': 'test.invalid'}, __file__)
+        session = create_session()
+        name = 'A test'
+        url = '/api/test'
+        test_spec = {
+            'name': name,
+            'url': url,
+            'max-diff': None,
+        }
+        assertions = {}
+
+        test = WebTest.from_dict(
+            session, test_spec, config, assertions,
+            self.test_parameter_plugins)
+
+        responses.add(
+            self._get_web_test_method(test),
+            test.url,
+            status=204,
+        )
+
+        # When
+        case = Mock()
+        test.run(case)
+
+        # Then
+        # maxDiff set to None
+        self.assertIs(case.maxDiff, None)
+
+    @responses.activate
+    def test_run_int_max_diff(self):
+        # Given
+        config = Config.from_dict({'host': 'test.invalid'}, __file__)
+        session = create_session()
+        name = 'A test'
+        url = '/api/test'
+        test_spec = {
+            'name': name,
+            'url': url,
+            'max-diff': 2121,
+        }
+        assertions = {}
+
+        test = WebTest.from_dict(
+            session, test_spec, config, assertions,
+            self.test_parameter_plugins)
+
+        responses.add(
+            self._get_web_test_method(test),
+            test.url,
+            status=204,
+        )
+
+        # When
+        case = Mock()
+        test.run(case)
+
+        # Then
+        # maxDiff set to value
+        self.assertEqual(case.maxDiff, 2121)
+
     def test_connection_error(self):
         # Given
         config = Config.from_dict({'host': 'test.invalid'}, __file__)
diff --git a/usagi/tests/test_yaml_test_loader.py b/usagi/tests/test_yaml_test_loader.py
index 1527b0e..0dcd22d 100644
--- a/usagi/tests/test_yaml_test_loader.py
+++ b/usagi/tests/test_yaml_test_loader.py
@@ -316,3 +316,69 @@ class TestYamlTestLoader(unittest.TestCase):
         self.assertEqual(test.request.url, 'http://test.domain/another')
         self.assertEqual(post_1.request.url, 'http://test.domain/one')
         self.assertEqual(post_2.request.url, 'http://test.domain/two')
+
+    def test_load_yaml_tests_case_max_diff_null(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+        ---
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              max-diff: null
+              tests:
+                - name: "Test root URL"
+                  url: "/"
+                - name: "Test sub URL"
+                  url: "/sub/"
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # When
+        suite = self.loader.load_tests_from_yaml(
+            test_data, '/path/to/foo.yaml')
+
+        # Then
+        self.assertIsInstance(suite, TestSuite)
+        self.assertEqual(suite.countTestCases(), 2)
+        cls1, cls2 = [type(case) for case in find_test_cases(suite)]
+        self.assertIs(cls1, cls2)
+        self.assertIsNone(cls1.maxDiff)
+
+    def test_load_yaml_tests_case_max_diff_number(self):
+        # Given
+        test_yaml = textwrap.dedent("""
+        ---
+          version: '1.0'
+
+          config:
+            host: test.domain
+
+          cases:
+            - name: "Basic"
+              max-diff: 1234
+              tests:
+                - name: "Test root URL"
+                  url: "/"
+                - name: "Test sub URL"
+                  url: "/sub/"
+
+        """)
+
+        test_data = yaml.safe_load(test_yaml)
+
+        # When
+        suite = self.loader.load_tests_from_yaml(
+            test_data, '/path/to/foo.yaml')
+
+        # Then
+        self.assertIsInstance(suite, TestSuite)
+        self.assertEqual(suite.countTestCases(), 2)
+        cls1, cls2 = [type(case) for case in find_test_cases(suite)]
+        self.assertIs(cls1, cls2)
+        self.assertEqual(cls1.maxDiff, 1234)
diff --git a/usagi/web_test.py b/usagi/web_test.py
index db205c6..b410e84 100644
--- a/usagi/web_test.py
+++ b/usagi/web_test.py
@@ -33,6 +33,9 @@ def initialize_test_parameter_loaders(test_parameter_plugins, parameter_specs):
         yield cls.from_dict({name: value})
 
 
+_Default = object()
+
+
 class WebTest(object):
     """The main entry-point into a single web test case.
 
@@ -42,7 +45,7 @@ class WebTest(object):
     """
 
     def __init__(self, session, config, name, path, assertions,
-                 parameter_loaders):
+                 parameter_loaders, max_diff):
         super(WebTest, self).__init__()
         self.session = session
         self.name = name
@@ -50,6 +53,7 @@ class WebTest(object):
         self.path = path
         self.assertions = assertions
         self.parameter_loaders = parameter_loaders
+        self.max_diff = max_diff
 
     @property
     def url(self):
@@ -80,6 +84,8 @@ class WebTest(object):
         spec = spec.copy()
         name = spec.pop('name')
 
+        max_diff = spec.pop('max-diff', _Default)
+
         assertion_specs = spec.pop('assertions', [])
         assertions = initialize_assertions(
             assertions_map, assertion_specs)
@@ -100,6 +106,7 @@ class WebTest(object):
             path=spec['url'],
             assertions=list(assertions),
             parameter_loaders=list(parameter_loaders),
+            max_diff=max_diff,
         )
 
     def run(self, case):
@@ -111,6 +118,9 @@ class WebTest(object):
             The ``TestCase`` instance used to record test results.
 
         """
+        if self.max_diff is not _Default:
+            case.maxDiff = self.max_diff
+
         try:
             url = self.url
         except InvalidVariableType as exc:
diff --git a/usagi/yaml_test_loader.py b/usagi/yaml_test_loader.py
index 373a7ca..5a2457f 100644
--- a/usagi/yaml_test_loader.py
+++ b/usagi/yaml_test_loader.py
@@ -98,6 +98,9 @@ def create_test_case_for_case(filename, config, case, assertions_map,
     )
     class_dict[TEST_NAME_ATTRIBUTE] = case['name']
 
+    if 'max-diff' in case:
+        class_dict['maxDiff'] = case['max-diff']
+
     def __str__(self):
         method = getattr(self, self._testMethodName)
         template = '{0!r} ({1})'