Repository: cdent/gabbi

Issue:
substitutions not allowed in left hand side of json paths in response_json_paths
``` yaml
$.allocations.$ENVIRON['RP_UUID'].DISK_GB: 40
```

does not work, maybe it should?


Comments:

Patch:
diff --git a/docs/source/format.rst b/docs/source/format.rst
index c12d3b2..de15bbb 100644
--- a/docs/source/format.rst
+++ b/docs/source/format.rst
@@ -274,7 +274,8 @@ All of these variables may be used in all of the following fields:
 * ``data``
 * ``request_headers``
 * ``response_strings``
-* ``response_json_paths`` (on the value side of the key value pair)
+* ``response_json_paths`` (in both the key and value, see
+  :ref:`json path substitution <json-subs>` for more info)
 * ``response_headers`` (on the value side of the key value pair)
 * ``response_forbidden_headers``
 * ``count`` and ``delay`` fields of ``poll``
diff --git a/docs/source/jsonpath.rst b/docs/source/jsonpath.rst
index 15d4e1d..317603b 100644
--- a/docs/source/jsonpath.rst
+++ b/docs/source/jsonpath.rst
@@ -64,8 +64,48 @@ lead to difficult to read tests and it also indicates that your
 gabbi tests are being used to test your serializers and data models,
 not just your API interactions.
 
+It is also possible to read raw JSON from disk for either all or
+some of a JSON response::
+
+    response_json_paths:
+        $: @<data.json
+
+or::
+
+    response_json_paths:
+        $.pets: <@pets.json
+        $.pets[0]: <@cat.json
+
+Examples like this can be found in one of gabbi's `own tests`_.
+
 There are more JSONPath examples in :doc:`example` and in the
 `jsonpath_rw`_ and `jsonpath_rw_ext`_ documentation.
 
+.. _json-subs:
+
+Substitution
+------------
+
+:ref:`Substitutions <state-substitution>` can be made in both the
+left (query) and right (expected) hand sides of the json path
+expression. When subtitutions are used in the query, care must be
+taken to ensure proper quoting of the resulting value. For example
+if there is a uuid (with hyphens) at ``$RESPONSE['$.id']`` then this
+expression may fail::
+
+    $.nested.structure.$RESPONSE['$.id'].name: foobar
+
+as it will evaluate to something like::
+
+   $.nested.structure.ADC8AAFC-D564-40D1-9724-7680D3C010C2.name: foobar
+
+which may be treated as an arithemtic expression by the json path
+parser. The test author should write::
+
+    $.nested.structure["$RESPONSE['$.id']"].name: foobar
+
+to quote the result of the substitution.
+
 .. _jsonpath_rw: http://jsonpath-rw.readthedocs.io/en/latest/
 .. _jsonpath_rw_ext: https://python-jsonpath-rw-ext.readthedocs.io/en/latest/
+.. _own tests: https://github.com/cdent/gabbi/blob/master/gabbi/tests/gabbits_intercept/data.yaml
diff --git a/gabbi/case.py b/gabbi/case.py
index d903dd5..79c18c9 100644
--- a/gabbi/case.py
+++ b/gabbi/case.py
@@ -174,6 +174,10 @@ class HTTPTestCase(testtools.TestCase):
 
         return message
 
+    def load_data_file(self, filename):
+        """Read a file from the current test directory."""
+        return self._load_data_file(filename)
+
     def _assert_response(self):
         """Compare the response with expected data."""
         self._test_status(self.test_data['status'], self.response['status'])
@@ -505,7 +509,7 @@ class HTTPTestCase(testtools.TestCase):
         """
         if isinstance(data, str):
             if data.startswith('<@'):
-                info = self._load_data_file(data.replace('<@', '', 1))
+                info = self.load_data_file(data.replace('<@', '', 1))
                 if utils.not_binary(content_type):
                     data = six.text_type(info, 'UTF-8')
                 else:
diff --git a/gabbi/handlers/jsonhandler.py b/gabbi/handlers/jsonhandler.py
index 5b45720..2f85360 100644
--- a/gabbi/handlers/jsonhandler.py
+++ b/gabbi/handlers/jsonhandler.py
@@ -14,6 +14,8 @@
 
 import json
 
+import six
+
 from gabbi.handlers import base
 from gabbi import json_parser
 
@@ -73,11 +75,8 @@ class JSONHandler(base.ContentHandler):
 
     def action(self, test, path, value=None):
         """Test json_paths against json data."""
-        # NOTE: This process has some advantages over other process that
-        # might come along because the JSON data has already been
-        # processed (to provided for the magic template replacing).
-        # Other handlers that want access to data structures will need
-        # to do their own processing.
+        # Do template expansion in the left hand side.
+        path = test.replace_template(path)
         try:
             match = self.extract_json_path_value(
                 test.response_data, path)
@@ -86,6 +85,13 @@ class JSONHandler(base.ContentHandler):
         except ValueError:
             raise AssertionError('json path %s cannot match %s' %
                                  (path, test.response_data))
+
+        # read data from disk if the value starts with '<@'
+        if isinstance(value, str) and value.startswith('<@'):
+            info = test.load_data_file(value.replace('<@', '', 1))
+            info = six.text_type(info, 'UTF-8')
+            value = self.loads(info)
+
         expected = test.replace_template(value)
         # If expected is a string, check to see if it is a regex.
         if (hasattr(expected, 'startswith') and expected.startswith('/')


Test patch:
diff --git a/gabbi/tests/gabbits_intercept/cat.json b/gabbi/tests/gabbits_intercept/cat.json
new file mode 100644
index 0000000..d27c320
--- /dev/null
+++ b/gabbi/tests/gabbits_intercept/cat.json
@@ -0,0 +1,4 @@
+{
+    "type": "cat",
+    "sound": "meow"
+}
diff --git a/gabbi/tests/gabbits_intercept/data.yaml b/gabbi/tests/gabbits_intercept/data.yaml
index 7bd1a7f..fbe832d 100644
--- a/gabbi/tests/gabbits_intercept/data.yaml
+++ b/gabbi/tests/gabbits_intercept/data.yaml
@@ -49,3 +49,26 @@ tests:
       request_headers:
           content-type: text/plain
       data: <@utf8.txt
+
+    - name: json value from disk
+      POST: /
+      request_headers:
+          content-type: application/json
+      data: <@data.json
+      response_json_paths:
+          foo['bár']: 1
+          $: <@data.json
+
+    - name: partial json from disk
+      POST: /
+      request_headers:
+          content-type: application/json
+      data:
+          pets:
+              - type: cat
+                sound: meow
+              - type: dog
+                sound: woof
+      response_json_paths:
+          $.pets: <@pets.json
+          $.pets[0]: <@cat.json
diff --git a/gabbi/tests/gabbits_intercept/json-left-side.yaml b/gabbi/tests/gabbits_intercept/json-left-side.yaml
new file mode 100644
index 0000000..6ce342c
--- /dev/null
+++ b/gabbi/tests/gabbits_intercept/json-left-side.yaml
@@ -0,0 +1,37 @@
+defaults:
+  request_headers:
+      content-type: application/json
+  verbose: True
+
+tests:
+- name: left side json one
+  desc: for reuse on the next test
+  POST: /
+  data:
+      alpha: alpha1
+      beta: beta1
+
+- name: expand left side
+  POST: /
+  data:
+      alpha1: alpha
+      beta1: beta
+  response_json_paths:
+      $["$RESPONSE['$.alpha']"]: alpha
+
+- name: expand environ left side
+  POST: /
+  data:
+      alpha1: alpha
+      beta1: beta
+      1: cow
+  response_json_paths:
+      $.['$ENVIRON['ONE']']: cow
+
+- name: set key and value
+  GET: /jsonator?key=$ENVIRON['ONE']&value=10
+
+- name: check key and value
+  GET: /jsonator?key=$ENVIRON['ONE']&value=10
+  response_json_paths:
+      $.["$ENVIRON['ONE']"]: $RESPONSE['$['1']']
diff --git a/gabbi/tests/gabbits_intercept/pets.json b/gabbi/tests/gabbits_intercept/pets.json
new file mode 100644
index 0000000..3c56db0
--- /dev/null
+++ b/gabbi/tests/gabbits_intercept/pets.json
@@ -0,0 +1,10 @@
+[
+    {
+        "type": "cat",
+        "sound": "meow"
+    },
+    {
+        "type": "dog",
+        "sound": "woof"
+    }
+]
diff --git a/gabbi/tests/simple_wsgi.py b/gabbi/tests/simple_wsgi.py
index fabb7e3..b7528ff 100644
--- a/gabbi/tests/simple_wsgi.py
+++ b/gabbi/tests/simple_wsgi.py
@@ -107,6 +107,11 @@ class SimpleWsgi(object):
             # fall through if we've ended the loop
         elif path_info == '/cookie':
             headers.append(('Set-Cookie', 'session=1234; domain=.example.com'))
+        elif path_info == '/jsonator':
+            json_data = json.dumps({query_data['key'][0]:
+                                    query_data['value'][0]})
+            start_response('200 OK', [('Content-Type', 'application/json')])
+            return [json_data.encode('utf-8')]
 
         start_response('200 OK', headers)
 
diff --git a/gabbi/tests/test_load_data_file.py b/gabbi/tests/test_load_data_file.py
index 99536d4..f3304a6 100644
--- a/gabbi/tests/test_load_data_file.py
+++ b/gabbi/tests/test_load_data_file.py
@@ -35,7 +35,7 @@ class DataFileTest(unittest.TestCase):
 
     def _assert_content_read(self, filepath):
         self.assertEqual(
-            'dummy content', self.http_case._load_data_file(filepath))
+            'dummy content', self.http_case.load_data_file(filepath))
 
     def test_load_file(self, m_open):
         self.http_case.test_directory = '.'
@@ -52,7 +52,7 @@ class DataFileTest(unittest.TestCase):
         filepath = '/top-level.private'
 
         with self.assertRaises(ValueError):
-            self.http_case._load_data_file(filepath)
+            self.http_case.load_data_file(filepath)
         self.assertFalse(m_open.called)
 
     def test_load_file_in_parent_dir(self, m_open):
@@ -60,7 +60,7 @@ class DataFileTest(unittest.TestCase):
         filepath = '../file-in-parent-dir.txt'
 
         with self.assertRaises(ValueError):
-            self.http_case._load_data_file(filepath)
+            self.http_case.load_data_file(filepath)
         self.assertFalse(m_open.called)
 
     def test_load_file_within_test_directory(self, m_open):
@@ -73,5 +73,5 @@ class DataFileTest(unittest.TestCase):
         self.http_case.test_directory = '/a/b/c'
         filepath = '../../b/E/file-in-test-dir.txt'
         with self.assertRaises(ValueError):
-            self.http_case._load_data_file(filepath)
+            self.http_case.load_data_file(filepath)
         self.assertFalse(m_open.called)