Repository: taverntesting/tavern

Issue:
question: how to express that ordering doesn't matter?
I have a test:

```
  - name: instance list
    request:
      url: http://localhost:10000/a1-p/policytypes/1006001/policies
      method: GET
    response:
      status_code: 200
      body: [qt1, qt2]
```

how do I express that `[qt1, qt2]` OR `[qt2,qt1]` is OK? Is there an "unordered_ok" or some equivalent flag?



Discussion:

Patch:
diff --git a/README.rst b/README.rst
index 51d2ec8..0ffe834 100644
--- a/README.rst
+++ b/README.rst
@@ -60,6 +60,9 @@ Then, let's create a basic test, ``test_minimal.tavern.yaml``:
           status_code: 200
           json:
             id: 1
+            userId: 1
+            title: "sunt aut facere repellat provident occaecati excepturi optio reprehenderit"
+            body: "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto"
 
 This file can have any name, but if you intend to use Pytest with
 Tavern, it will only pick up files called ``test_*.tavern.yaml``.
diff --git a/docs/source/basics.md b/docs/source/basics.md
index 37d6e79..743bb21 100644
--- a/docs/source/basics.md
+++ b/docs/source/basics.md
@@ -17,6 +17,9 @@ stages:
       status_code: 200
       json:
         id: 1
+        userId: 1
+        title: "sunt aut facere repellat provident occaecati excepturi optio reprehenderit"
+        body: "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto"
       save:
         json:
           returned_id: id
diff --git a/docs/source/examples.md b/docs/source/examples.md
index 288b403..ac036e4 100644
--- a/docs/source/examples.md
+++ b/docs/source/examples.md
@@ -16,6 +16,9 @@ stages:
       status_code: 200
       json:
         id: 1
+        userId: 1
+        title: "sunt aut facere repellat provident occaecati excepturi optio reprehenderit"
+        body: "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto"
 ```
 
 Next, install Tavern if you have not already:
diff --git a/smoke.bash b/smoke.bash
index 7d94949..e579889 100755
--- a/smoke.bash
+++ b/smoke.bash
@@ -2,7 +2,7 @@
 
 set -ex
 
-PYVER=38
+PYVER=39
 
 tox -c tox.ini        \
   -e py${PYVER}flakes \
diff --git a/tavern/_plugins/mqtt/response.py b/tavern/_plugins/mqtt/response.py
index 52d451c..d7e3f50 100644
--- a/tavern/_plugins/mqtt/response.py
+++ b/tavern/_plugins/mqtt/response.py
@@ -59,7 +59,7 @@ class MQTTResponse(BaseResponse):
         timeout = self.expected.get("timeout", 1)
 
         test_strictness = self.test_block_config["strict"]
-        block_strictness = test_strictness.setting_for("json").is_on()
+        block_strictness = test_strictness.setting_for("json")
 
         expected_payload, expect_json_payload = self._get_payload_vals()
 
diff --git a/tavern/_plugins/rest/response.py b/tavern/_plugins/rest/response.py
index 6a39830..e97d3c0 100644
--- a/tavern/_plugins/rest/response.py
+++ b/tavern/_plugins/rest/response.py
@@ -232,5 +232,5 @@ class RestResponse(BaseResponse):
         logger.debug("Validating response %s against %s", blockname, expected_block)
 
         test_strictness = self.test_block_config["strict"]
-        block_strictness = test_strictness.setting_for(blockname).is_on()
+        block_strictness = test_strictness.setting_for(blockname)
         self.recurse_check_key_match(expected_block, block, blockname, block_strictness)
diff --git a/tavern/response/base.py b/tavern/response/base.py
index 6a0d214..3717009 100644
--- a/tavern/response/base.py
+++ b/tavern/response/base.py
@@ -56,6 +56,7 @@ class BaseResponse(object):
             Optionally use a validation library too
 
         Args:
+            strict: strictness setting for this block
             expected_block (dict): expected data
             block (dict): actual data
             blockname (str): 'name' of this block (params, mqtt, etc) for error messages
diff --git a/tavern/util/dict_util.py b/tavern/util/dict_util.py
index d2c776c..bb4ec25 100644
--- a/tavern/util/dict_util.py
+++ b/tavern/util/dict_util.py
@@ -17,6 +17,7 @@ from tavern.util.loader import (
 
 from . import exceptions
 from .formatted_str import FormattedString
+from .strict_util import StrictSetting, extract_strict_setting
 
 logger = logging.getLogger(__name__)
 
@@ -350,7 +351,7 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
         KeyMismatchError: expected_val and actual_val did not match
     """
 
-    # pylint: disable=too-many-locals,too-many-statements
+    # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks
 
     def full_err():
         """Get error in the format:
@@ -395,6 +396,8 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
             issubclass(actual_type, type(expected_val))
         )
 
+    strict, strict_setting = extract_strict_setting(strict)
+
     try:
         assert actual_val == expected_val
     except AssertionError as e:
@@ -455,7 +458,7 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
             for key in to_recurse:
                 try:
                     check_keys_match_recursive(
-                        expected_val[key], actual_val[key], keys + [key], strict
+                        expected_val[key], actual_val[key], keys + [key], strict_setting
                     )
                 except KeyError:
                     logger.debug(
@@ -489,7 +492,7 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
                         # Found one - check if it matches
                         try:
                             check_keys_match_recursive(
-                                e_val, current_response_val, keys + [i], strict
+                                e_val, current_response_val, keys + [i], strict_setting
                             )
                         except exceptions.KeyMismatchError:
                             # Doesn't match what we're looking for
@@ -500,6 +503,8 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
                             )
                         else:
                             logger.debug("'%s' present in response", e_val)
+                            if strict_setting == StrictSetting.LIST_ANY_ORDER:
+                                actual_iter = iter(actual_val)
                             break
 
                 if missing:
@@ -517,11 +522,13 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
 
                 for i, (e_val, a_val) in enumerate(zip(expected_val, actual_val)):
                     try:
-                        check_keys_match_recursive(e_val, a_val, keys + [i], strict)
+                        check_keys_match_recursive(
+                            e_val, a_val, keys + [i], strict_setting
+                        )
                     except exceptions.KeyMismatchError as sub_e:
-                        # This should _ALWAYS_ raise an error, but it will be more
-                        # obvious where the error came from (in python 3 at least)
-                        # and will take ANYTHING into account
+                        # This should _ALWAYS_ raise an error (unless the reason it didn't match was the
+                        # 'anything' sentinel), but it will be more obvious where the error came from
+                        # (in python 3 at least) and will take ANYTHING into account
                         raise sub_e from e
         elif expected_val is ANYTHING:
             logger.debug("Actual value = '%s' - matches !anything", actual_val)
diff --git a/tavern/util/strict_util.py b/tavern/util/strict_util.py
index 323c548..638f5ec 100644
--- a/tavern/util/strict_util.py
+++ b/tavern/util/strict_util.py
@@ -1,52 +1,65 @@
 from distutils.util import strtobool
 import enum
+import logging
 import re
 
 import attr
 
 from tavern.util import exceptions
 
+logger = logging.getLogger(__name__)
 
-class _StrictSetting(enum.Enum):
+
+class StrictSetting(enum.Enum):
     ON = 1
     OFF = 2
     UNSET = 3
+    LIST_ANY_ORDER = 4
 
 
 valid_keys = ["json", "headers", "redirect_query_params"]
 
+valid_switches = ["on", "off", "list_any_order"]
+
 
-def setting_factory(str_setting):
+def strict_setting_factory(str_setting):
     """Converts from cmdline/setting file to an enum"""
     if str_setting is None:
-        return _StrictSetting.UNSET
+        return StrictSetting.UNSET
     else:
+        if str_setting == "list_any_order":
+            return StrictSetting.LIST_ANY_ORDER
+
         parsed = strtobool(str_setting)
 
         if parsed:
-            return _StrictSetting.ON
+            return StrictSetting.ON
         else:
-            return _StrictSetting.OFF
+            return StrictSetting.OFF
 
 
 @attr.s(frozen=True)
-class _StrictOption:
+class StrictOption:
     section = attr.ib(type=str)
-    setting = attr.ib(type=_StrictSetting)
+    setting = attr.ib(type=StrictSetting)
 
     def is_on(self):
         if self.section == "json":
             # Must be specifically disabled for response body
-            return self.setting != _StrictSetting.OFF
+            return self.setting not in [StrictSetting.OFF, StrictSetting.LIST_ANY_ORDER]
         else:
             # Off by default for everything else
-            return self.setting == _StrictSetting.ON
+            return self.setting in [StrictSetting.ON]
 
 
 def validate_and_parse_option(key):
-    regex = r"(?P<section>{})(:(?P<setting>on|off))?".format("|".join(valid_keys))
+    regex = re.compile(
+        "(?P<section>{sections})(:(?P<setting>{switches}))?".format(
+            sections="|".join(valid_keys), switches="|".join(valid_switches)
+        )
+    )
 
-    match = re.fullmatch(regex, key)
+    match = regex.fullmatch(key)
 
     if not match:
         raise exceptions.InvalidConfigurationException(
@@ -56,15 +69,21 @@ def validate_and_parse_option(key):
         )
 
     as_dict = match.groupdict()
-    return _StrictOption(as_dict["section"], setting_factory(as_dict["setting"]))
+
+    if as_dict["section"] != "json" and as_dict["setting"] == "list_any_order":
+        logger.warning(
+            "Using 'list_any_order' key outside of 'json' section has no meaning"
+        )
+
+    return StrictOption(as_dict["section"], strict_setting_factory(as_dict["setting"]))
 
 
 @attr.s(frozen=True)
 class StrictLevel:
-    json = attr.ib(default=_StrictOption("json", setting_factory(None)))
-    headers = attr.ib(default=_StrictOption("headers", setting_factory(None)))
+    json = attr.ib(default=StrictOption("json", strict_setting_factory(None)))
+    headers = attr.ib(default=StrictOption("headers", strict_setting_factory(None)))
     redirect_query_params = attr.ib(
-        default=_StrictOption("redirect_query_params", setting_factory(None))
+        default=StrictOption("redirect_query_params", strict_setting_factory(None))
     )
 
     @classmethod
@@ -96,3 +115,26 @@ class StrictLevel:
     @classmethod
     def all_off(cls):
         return cls.from_options([i + ":off" for i in valid_keys])
+
+
+def extract_strict_setting(strict):
+    """Takes either a bool, StrictOption, or a StrictSetting and return the bool representation and StrictSetting representation"""
+    if isinstance(strict, StrictSetting):
+        strict_setting = strict
+        strict = strict == StrictSetting.ON
+    elif isinstance(strict, StrictOption):
+        strict_setting = strict.setting
+        strict = strict.is_on()
+    elif isinstance(strict, bool):
+        strict_setting = strict_setting_factory(str(strict))
+    elif strict is None:
+        strict = False
+        strict_setting = strict_setting_factory("false")
+    else:
+        raise exceptions.InvalidConfigurationException(
+            "Unable to parse strict setting '{}' of type '{}'".format(
+                strict, type(strict)
+            )
+        )
+
+    return strict, strict_setting


Test patch:
diff --git a/tests/integration/test_minimal.tavern.yaml b/tests/integration/test_minimal.tavern.yaml
new file mode 100644
index 0000000..fc3fcc4
--- /dev/null
+++ b/tests/integration/test_minimal.tavern.yaml
@@ -0,0 +1,21 @@
+---
+# Every test file has one or more tests...
+test_name: Get some fake data from the JSON placeholder API
+
+# ...and each test has one or more stages (e.g. an HTTP request)
+stages:
+  - name: Make sure we have the right ID
+
+    # Define the request to be made...
+    request:
+      url: https://jsonplaceholder.typicode.com/posts/1
+      method: GET
+
+    # ...and the expected response code and body
+    response:
+      status_code: 200
+      json:
+        id: 1
+        userId: 1
+        title: "sunt aut facere repellat provident occaecati excepturi optio reprehenderit"
+        body: "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto"
\ No newline at end of file
diff --git a/tests/integration/test_strict_key_checks.tavern.yaml b/tests/integration/test_strict_key_checks.tavern.yaml
index 8b0f07c..6f273fa 100644
--- a/tests/integration/test_strict_key_checks.tavern.yaml
+++ b/tests/integration/test_strict_key_checks.tavern.yaml
@@ -319,7 +319,7 @@ stages:
       url: "{host}/fake_list"
     response:
       status_code: 200
-#      Use new syntax
+      #      Use new syntax
       strict:
         - json:off
       json:
@@ -422,6 +422,95 @@ stages:
 
 ---
 
+test_name: Test matching any order on json
+
+strict:
+  - json:list_any_order
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/fake_list"
+    response:
+      status_code: 200
+      json:
+        - 2
+        - c
+        - a
+        - -3.0
+        - 1
+
+---
+
+test_name: Test matching any order on json
+
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/fake_list"
+    response:
+      strict:
+        - json:list_any_order
+      status_code: 200
+      json:
+        - 2
+        - c
+        - a
+        - -3.0
+        - 1
+
+---
+
+test_name: Test matching any order on json nested
+
+strict:
+  - json:list_any_order
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/nested_list"
+    response:
+      status_code: 200
+      json:
+        top:
+        - b
+        - key: val
+        - a
+
+---
+
+test_name: Test matching any order on json nested
+
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/nested_list"
+    response:
+      strict:
+        - json:list_any_order
+      status_code: 200
+      json:
+        top:
+        - b
+        - key: val
+
+---
+
 test_name: Test non-strict key matching in one stage does not leak over to the next
 
 _xfail: run
diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py
index 85048b0..a5c41b3 100644
--- a/tests/unit/test_helpers.py
+++ b/tests/unit/test_helpers.py
@@ -21,7 +21,7 @@ from tavern.util.dict_util import _check_and_format_values, format_keys
 from tavern.util.loader import ForceIncludeToken
 from tavern.util.strict_util import (
     StrictLevel,
-    _StrictSetting,
+    StrictSetting,
     validate_and_parse_option,
 )
 
@@ -393,18 +393,18 @@ class TestStrictUtils:
     def test_set_on(self, section):
         level = StrictLevel.from_options([section + ":on"])
 
-        assert level.setting_for(section).setting == _StrictSetting.ON
+        assert level.setting_for(section).setting == StrictSetting.ON
         assert level.setting_for(section).is_on()
 
     @pytest.mark.parametrize("section", ["json", "headers", "redirect_query_params"])
     def test_set_off(self, section):
         level = StrictLevel.from_options([section + ":off"])
 
-        assert level.setting_for(section).setting == _StrictSetting.OFF
+        assert level.setting_for(section).setting == StrictSetting.OFF
         assert not level.setting_for(section).is_on()
 
     @pytest.mark.parametrize("section", ["json", "headers", "redirect_query_params"])
     def test_unset(self, section):
         level = StrictLevel.from_options([section])
 
-        assert level.setting_for(section).setting == _StrictSetting.UNSET
+        assert level.setting_for(section).setting == StrictSetting.UNSET
diff --git a/tests/unit/test_strict_util.py b/tests/unit/test_strict_util.py
new file mode 100644
index 0000000..3e58cba
--- /dev/null
+++ b/tests/unit/test_strict_util.py
@@ -0,0 +1,27 @@
+import pytest
+
+from tavern.util.strict_util import StrictOption, StrictSetting, extract_strict_setting
+
+
+@pytest.mark.parametrize(
+    "strict", [True, StrictSetting.ON, StrictOption("json", StrictSetting.ON)]
+)
+def test_extract_strict_setting_true(strict):
+    as_bool, as_setting = extract_strict_setting(strict)
+    assert as_bool is True
+
+
+@pytest.mark.parametrize(
+    "strict",
+    [
+        False,
+        StrictSetting.OFF,
+        StrictSetting.LIST_ANY_ORDER,
+        StrictSetting.UNSET,
+        StrictOption("json", StrictSetting.OFF),
+        None,
+    ],
+)
+def test_extract_strict_setting_false(strict):
+    as_bool, as_setting = extract_strict_setting(strict)
+    assert as_bool is False