Repository: kevin1024/vcrpy

Issue:
Error with body matcher for json, xmlrpc and form urlencoded
This is a tricky issue I encountered when using the body matcher on xmlrpc requests.

Symptoms: Sometimes the request won't match, sometimes it will, and this only affects certain requests. This occurs on Python 3.4 and I believe other python 3 versions but not on 2.7

Cause: An XMLRPC request has a body with XML inside which is generated from the parameters passed to the function call. Some parameters can be of dict type. xmlrpclib (or xmlrpc.client) will loop over items of the dict and generate the appropriate XML which will be the body of our POST request. Now items order is not guaranteed in dict and the behavior changed in python 3 such that the order of the same dict *can change anytime* and is not *more or less constant on the same computer* as in python 2. So the generated XML won't be necessarily the same as the one you recorded.

Fix suggestion: A custom xmlrpc body matcher that takes that into account and will compare the `struct` XML elements in the correct order.

The gzip compression didn't help me debuging this as I couldn't even read directly the cassettes...

Discussion:
Diaoul: No unit test because dict order is not deterministic.
Diaoul: More details in the commit message.

Patch:
diff --git a/README.rst b/README.rst
index a5dcd90..0b85862 100644
--- a/README.rst
+++ b/README.rst
@@ -5,7 +5,6 @@ VCR.py
    :alt: vcr.py
 
    vcr.py
-
 This is a Python version of `Ruby's VCR
 library <https://github.com/vcr/vcr>`__.
 
@@ -145,7 +144,9 @@ The following options are available :
 -  port (the port of the server receiving the request)
 -  path (the path of the request)
 -  query (the query string of the request)
--  body (the entire request body)
+-  raw\_body (the entire request body as is)
+-  body (the entire request body unmarshalled by content-type
+   i.e. xmlrpc, json, form-urlencoded, falling back on raw\_body)
 -  headers (the headers of the request)
 
    Backwards compatible matchers:
diff --git a/setup.py b/setup.py
index 4b8c549..eaf20c3 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,6 @@
 import sys
 from setuptools import setup, find_packages
 from setuptools.command.test import test as TestCommand
-import pkg_resources
 
 long_description = open('README.rst', 'r').read()
 
@@ -21,21 +20,6 @@ class PyTest(TestCommand):
         sys.exit(errno)
 
 
-install_requires=['PyYAML', 'wrapt', 'six>=1.5']
-
-
-extras_require = {
-    ':python_version in "2.4, 2.5, 2.6"':
-        ['contextlib2', 'backport_collections', 'mock'],
-    ':python_version in "2.7, 3.1, 3.2"': ['contextlib2', 'mock'],
-}
-
-
-if 'bdist_wheel' not in sys.argv:
-    for key, value in extras_require.items():
-        if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]):
-            install_requires.extend(value)
-
 setup(
     name='vcrpy',
     version='1.6.0',
@@ -48,8 +32,12 @@ setup(
     author_email='me@kevinmccarthy.org',
     url='https://github.com/kevin1024/vcrpy',
     packages=find_packages(exclude=("tests*",)),
-    install_requires=install_requires,
-    extras_require=extras_require,
+    install_requires=['PyYAML', 'wrapt', 'six>=1.5'],
+    extras_require = {
+        ':python_version in "2.4, 2.5, 2.6"':
+            ['contextlib2', 'backport_collections', 'mock'],
+        ':python_version in "2.7, 3.1, 3.2"': ['contextlib2', 'mock'],
+    },
     license='MIT',
     tests_require=['pytest', 'mock', 'pytest-localserver'],
     cmdclass={'test': PyTest},
diff --git a/vcr/config.py b/vcr/config.py
index 1faef3b..7655a3a 100644
--- a/vcr/config.py
+++ b/vcr/config.py
@@ -47,6 +47,7 @@ class VCR(object):
             'path': matchers.path,
             'query': matchers.query,
             'headers': matchers.headers,
+            'raw_body': matchers.raw_body,
             'body': matchers.body,
         }
         self.record_mode = record_mode
diff --git a/vcr/matchers.py b/vcr/matchers.py
index 91bce11..39c5949 100644
--- a/vcr/matchers.py
+++ b/vcr/matchers.py
@@ -1,3 +1,6 @@
+import json
+from six.moves import urllib, xmlrpc_client
+from .util import CaseInsensitiveDict, read_body
 import logging
 log = logging.getLogger(__name__)
 
@@ -30,10 +33,23 @@ def query(r1, r2):
     return r1.query == r2.query
 
 
+def raw_body(r1, r2):
+    return read_body(r1) == read_body(r2)
+
+
 def body(r1, r2):
-    if hasattr(r1.body, 'read') and hasattr(r2.body, 'read'):
-        return r1.body.read() == r2.body.read()
-    return r1.body == r2.body
+    r1_body = read_body(r1)
+    r2_body  = read_body(r2)
+    r1_headers = CaseInsensitiveDict(r1.headers)
+    r2_headers = CaseInsensitiveDict(r2.headers)
+    if r1_headers.get('Content-Type') == r2_headers.get('Content-Type') == 'application/x-www-form-urlencoded':
+        return urllib.parse.parse_qs(r1_body) == urllib.parse.parse_qs(r2_body)
+    if r1_headers.get('Content-Type') == r2_headers.get('Content-Type') == 'application/json':
+        return json.loads(r1_body) == json.loads(r2_body)
+    if ('xmlrpc' in r1_headers.get('User-Agent', '') and 'xmlrpc' in r2_headers.get('User-Agent', '') and
+        r1_headers.get('Content-Type') == r2_headers.get('Content-Type') == 'text/xml'):
+        return xmlrpc_client.loads(r1_body) == xmlrpc_client.loads(r2_body)
+    return r1_body == r2_body
 
 
 def headers(r1, r2):
diff --git a/vcr/util.py b/vcr/util.py
index 57f72b1..8c5bd94 100644
--- a/vcr/util.py
+++ b/vcr/util.py
@@ -1,3 +1,74 @@
+import collections
+
+# Shamelessly stolen from https://github.com/kennethreitz/requests/blob/master/requests/structures.py
+class CaseInsensitiveDict(collections.MutableMapping):
+    """
+    A case-insensitive ``dict``-like object.
+    Implements all methods and operations of
+    ``collections.MutableMapping`` as well as dict's ``copy``. Also
+    provides ``lower_items``.
+    All keys are expected to be strings. The structure remembers the
+    case of the last key to be set, and ``iter(instance)``,
+    ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
+    will contain case-sensitive keys. However, querying and contains
+    testing is case insensitive::
+        cid = CaseInsensitiveDict()
+        cid['Accept'] = 'application/json'
+        cid['aCCEPT'] == 'application/json'  # True
+        list(cid) == ['Accept']  # True
+    For example, ``headers['content-encoding']`` will return the
+    value of a ``'Content-Encoding'`` response header, regardless
+    of how the header name was originally stored.
+    If the constructor, ``.update``, or equality comparison
+    operations are given keys that have equal ``.lower()``s, the
+    behavior is undefined.
+    """
+    def __init__(self, data=None, **kwargs):
+        self._store = dict()
+        if data is None:
+            data = {}
+        self.update(data, **kwargs)
+
+    def __setitem__(self, key, value):
+        # Use the lowercased key for lookups, but store the actual
+        # key alongside the value.
+        self._store[key.lower()] = (key, value)
+
+    def __getitem__(self, key):
+        return self._store[key.lower()][1]
+
+    def __delitem__(self, key):
+        del self._store[key.lower()]
+
+    def __iter__(self):
+        return (casedkey for casedkey, mappedvalue in self._store.values())
+
+    def __len__(self):
+        return len(self._store)
+
+    def lower_items(self):
+        """Like iteritems(), but with all lowercase keys."""
+        return (
+            (lowerkey, keyval[1])
+            for (lowerkey, keyval)
+            in self._store.items()
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, collections.Mapping):
+            other = CaseInsensitiveDict(other)
+        else:
+            return NotImplemented
+        # Compare insensitively
+        return dict(self.lower_items()) == dict(other.lower_items())
+
+    # Copy is required
+    def copy(self):
+        return CaseInsensitiveDict(self._store.values())
+
+    def __repr__(self):
+        return str(dict(self.items()))
+
 def partition_dict(predicate, dictionary):
     true_dict = {}
     false_dict = {}
@@ -14,3 +85,8 @@ def compose(*functions):
             res = function(res)
         return res
     return composed
+
+def read_body(request):
+    if hasattr(request.body, 'read'):
+        return request.body.read()
+    return request.body


Test patch:
diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py
index f942bd2..d4b32de 100644
--- a/tests/unit/test_matchers.py
+++ b/tests/unit/test_matchers.py
@@ -35,6 +35,38 @@ def test_uri_matcher():
             assert matched
 
 
+def test_body_matcher():
+    # raw
+    req1 = request.Request('POST', 'http://host.com/', '123', {})
+    req2 = request.Request('POST', 'http://another-host.com/', '123', {'Some-Header': 'value'})
+    assert matchers.body(req1, req2)
+
+    # application/x-www-form-urlencoded
+    req1 = request.Request('POST', 'http://host.com/', 'a=1&b=2', {'Content-Type': 'application/x-www-form-urlencoded'})
+    req2 = request.Request('POST', 'http://host.com/', 'b=2&a=1', {'Content-Type': 'application/x-www-form-urlencoded'})
+    assert matchers.body(req1, req2)
+
+    # application/json
+    req1 = request.Request('POST', 'http://host.com/', '{"a": 1, "b": 2}', {'Content-Type': 'application/json'})
+    req2 = request.Request('POST', 'http://host.com/', '{"b": 2, "a": 1}', {'content-type': 'application/json'})
+    assert matchers.body(req1, req2)
+
+    # xmlrpc
+    req1_body = (b"<?xml version='1.0'?><methodCall><methodName>test</methodName>"
+                 b"<params><param><value><array><data><value><struct>"
+                 b"<member><name>a</name><value><string>1</string></value></member>"
+                 b"<member><name>b</name><value><string>2</string></value></member>"
+                 b"</struct></value></data></array></value></param></params></methodCall>")
+    req2_body = (b"<?xml version='1.0'?><methodCall><methodName>test</methodName>"
+                 b"<params><param><value><array><data><value><struct>"
+                 b"<member><name>b</name><value><string>2</string></value></member>"
+                 b"<member><name>a</name><value><string>1</string></value></member>"
+                 b"</struct></value></data></array></value></param></params></methodCall>")
+    req1 = request.Request('POST', 'http://host.com/', req1_body, {'User-Agent': 'xmlrpclib', 'Content-Type': 'text/xml'})
+    req2 = request.Request('POST', 'http://host.com/', req2_body, {'user-agent': 'somexmlrpc', 'content-type': 'text/xml'})
+    assert matchers.body(req1, req2)
+
+
 def test_query_matcher():
     req1 = request.Request('GET', 'http://host.com/?a=b&c=d', '', {})
     req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {})