Repository: ethereum/web3.py

Issue:
Better error message on failed encoding
* Version: 4.1.0

### What was wrong?

A field of type `HexBytes` slipped through the pythonic middleware and the web3 provider tried to encode it with json. The only error message was:

> TypeError: Object of type 'HexBytes' is not JSON serializable


### How can it be fixed?

Catch type errors in:

```
~/web3/providers/base.py in encode_rpc_request(self, method, params)
     65             "method": method,
---> 66             "params": params or [],
     67             "id": next(self.request_counter),
     68         }))
     69
```
In case of a `TypeError`, inspect `params` and raise another `TypeError` saying which parameter index was failing to json encode, and (if applicable) which key in the `dict` at that index.


Discussion:
@carver i should be able to work on  #770 & #771 this week if you're still looking for help. 
Sure, that would be great! If you can only get to one, #770 has bigger impact
@carver  like so?
```python
def encode_rpc_request(self, method, params):
        try:
            return to_bytes(text=json.dumps({
                "jsonrpc": "2.0",
                "method": method,
                "params": params or [],
                "id": next(self.request_counter),
            }))
        except TypeError:
            err_msg = ''
            for i, p in enumerate(params):
                try:
                    json.dumps(p)
                except Exception:
                    err_msg += ' ({},{});'.format(i, repr(p))
            msg = 'encode_rpc_request failure for method {} for invalid (index, parameter): '
            msg += err_msg.rstrip(';') + '.'
            raise TypeError(msg.format(method))
```

Resolving patch:
diff --git a/web3/providers/__init__.py b/web3/providers/__init__.py
--- a/web3/providers/__init__.py
+++ b/web3/providers/__init__.py
@@ -1,6 +1,8 @@
 from .base import (  # noqa: F401
     BaseProvider,
+    JSONBaseProvider,
 )
+
 from .rpc import HTTPProvider  # noqa: F401
 from .ipc import IPCProvider  # noqa: F401
 from .websocket import WebsocketProvider  # noqa: F401
diff --git a/web3/providers/base.py b/web3/providers/base.py
--- a/web3/providers/base.py
+++ b/web3/providers/base.py
@@ -1,5 +1,4 @@
 import itertools
-import json
 
 from eth_utils import (
     to_bytes,
@@ -9,6 +8,9 @@
 from web3.middleware import (
     combine_middlewares,
 )
+from web3.utils.encoding import (
+    FriendlyJsonSerde,
+)
 
 
 class BaseProvider:
@@ -57,15 +59,18 @@ def __init__(self):
         self.request_counter = itertools.count()
 
     def decode_rpc_response(self, response):
-        return json.loads(to_text(response))
+        text_response = to_text(response)
+        return FriendlyJsonSerde().json_decode(text_response)
 
     def encode_rpc_request(self, method, params):
-        return to_bytes(text=json.dumps({
+        rpc_dict = {
             "jsonrpc": "2.0",
             "method": method,
             "params": params or [],
             "id": next(self.request_counter),
-        }))
+        }
+        encoded = FriendlyJsonSerde().json_encode(rpc_dict)
+        return to_bytes(text=encoded)
 
     def isConnected(self):
         try:
diff --git a/web3/utils/encoding.py b/web3/utils/encoding.py
--- a/web3/utils/encoding.py
+++ b/web3/utils/encoding.py
@@ -1,4 +1,5 @@
 # String encodings and numeric representations
+import json
 import re
 
 from eth_utils import (
@@ -11,6 +12,7 @@
     is_bytes,
     is_hex,
     is_integer,
+    is_list_like,
     remove_0x_prefix,
     to_hex,
 )
@@ -209,3 +211,54 @@ def hexstr_if_str(to_type, hexstr_or_primitive):
     else:
         (primitive, hexstr) = (hexstr_or_primitive, None)
     return to_type(primitive, hexstr=hexstr)
+
+
+class FriendlyJsonSerde:
+    '''
+    Friendly JSON serializer & deserializer
+
+    When encoding or decoding fails, this class collects
+    information on which fields failed, to show more
+    helpful information in the raised error messages.
+    '''
+    def _json_mapping_errors(self, mapping):
+        for key, val in mapping.items():
+            try:
+                self._friendly_json_encode(val)
+            except TypeError as exc:
+                yield "%r: because (%s)" % (key, exc)
+
+    def _json_list_errors(self, iterable):
+        for index, element in enumerate(iterable):
+            try:
+                self._friendly_json_encode(element)
+            except TypeError as exc:
+                yield "%d: because (%s)" % (index, exc)
+
+    def _friendly_json_encode(self, obj):
+        try:
+            encoded = json.dumps(obj)
+            return encoded
+        except TypeError as full_exception:
+            if hasattr(obj, 'items'):
+                item_errors = '; '.join(self._json_mapping_errors(obj))
+                raise TypeError("dict had unencodable value at keys: {{{}}}".format(item_errors))
+            elif is_list_like(obj):
+                element_errors = '; '.join(self._json_list_errors(obj))
+                raise TypeError("list had unencodable value at index: [{}]".format(element_errors))
+            else:
+                raise full_exception
+
+    def json_decode(self, json_str):
+        try:
+            decoded = json.loads(json_str)
+            return decoded
+        except json.decoder.JSONDecodeError as exc:
+            err_msg = 'Could not decode {} because of {}.'.format(repr(json_str), exc)
+            raise ValueError(err_msg)
+
+    def json_encode(self, obj):
+        try:
+            return self._friendly_json_encode(obj)
+        except TypeError as exc:
+            raise TypeError("Could not encode to JSON: {}".format(exc))


Test patch:
diff --git a/tests/core/utilities/test_encoding.py b/tests/core/utilities/test_encoding.py
--- a/tests/core/utilities/test_encoding.py
+++ b/tests/core/utilities/test_encoding.py
@@ -1,5 +1,7 @@
-# encoding: utf-8
-
+from ast import (
+    literal_eval,
+)
+import datetime
 import pytest
 from unittest.mock import (
     Mock,
@@ -14,7 +16,11 @@
     strategies as st,
 )
 
+from web3.providers import (
+    JSONBaseProvider,
+)
 from web3.utils.encoding import (
+    FriendlyJsonSerde as FriendlyJson,
     hex_encode_abi_type,
     hexstr_if_str,
     text_if_str,
@@ -141,3 +147,91 @@ def test_text_if_str_on_text(val):
     to_type = Mock(return_value='zoot')
     assert text_if_str(to_type, val) == 'zoot'
     assert to_type.call_args == ((None, ), {'text': val})
+
+
+@pytest.mark.parametrize(
+    "py_obj, exc_type, expected",
+    (
+        (
+            {
+                'date': [
+                    datetime.datetime(2018, 5, 10, 1, 5, 10).isoformat(),
+                    datetime.datetime(2018, 5, 10, 1, 5, 10).isoformat(),
+                ],
+                'other_date': datetime.datetime(2018, 5, 10, 1, 5, 10).date().isoformat(),
+            },
+            None,
+            '{"date": ["2018-05-10T01:05:10", "2018-05-10T01:05:10"], "other_date": "2018-05-10"}',
+        ),
+        (
+            {
+                'date': [datetime.datetime.utcnow(), datetime.datetime.now()],
+                'other_date': datetime.datetime.utcnow().date(),
+            },
+            TypeError,
+            "Could not encode to JSON: .*'other_date'.*is not JSON serializable",
+        ),
+    ),
+)
+def test_friendly_json_encode(py_obj, exc_type, expected):
+    if exc_type is None:
+        assert literal_eval(FriendlyJson().json_encode(py_obj)) == literal_eval(expected)
+    else:
+        with pytest.raises(exc_type, match=expected):
+            FriendlyJson().json_encode(py_obj)
+
+
+@pytest.mark.parametrize(
+    "json_str, expected",
+    (
+        (
+            '{"date": ["2018-05-10T01:05:10", "2018-05-10T01:05:10"],"other_date": "2018-05-10"}',
+            dict,
+        ),
+    ),
+)
+def test_friendly_json_decode(json_str, expected):
+    assert isinstance(FriendlyJson().json_decode(json_str), expected)
+
+
+@pytest.mark.parametrize(
+    "rpc_response, expected",
+    (
+        (
+            '{"jsonrpc": "2.0", "method": "test_method", "params": [], "id": 1}',
+            {"jsonrpc": "2.0", "method": "test_method", "params": [], "id": 1},
+        ),
+    ),
+)
+def test_decode_rpc_response(rpc_response, expected):
+    assert JSONBaseProvider().decode_rpc_response(rpc_response.encode('utf8')) == expected
+
+
+@pytest.mark.parametrize(
+    "rpc_kwargs, exc_type, expected",
+    (
+        (
+            {'id': 1, 'method': 'test', 'params': [], "jsonrpc": "2.0"},
+            None,
+            '{"id": 0, "method": "test", "params": [], "jsonrpc": "2.0",}',
+        ),
+        (
+            {'id': 0, 'method': 'test', 'params': [datetime.datetime(2018, 5, 10, 1, 5, 10)]},
+            TypeError,
+            r"Could not encode to JSON: .*'params'.* is not JSON serializable",
+        ),
+    ),
+)
+def test_encode_rpc_request(rpc_kwargs, exc_type, expected):
+    if exc_type is None:
+        res = JSONBaseProvider().encode_rpc_request(
+            rpc_kwargs['method'],
+            rpc_kwargs['params']
+        )
+        assert literal_eval(res.decode('utf8')) == literal_eval(expected)
+    else:
+        with pytest.raises(exc_type, match=expected):
+            JSONBaseProvider().encode_rpc_request(
+                rpc_kwargs['method'],
+                rpc_kwargs['params'],
+            )