Repository: pandas-dev/pandas

Issue:
Better error message when handling bad keys in json data
Should read_json check for bad keys at the top level and catch them instead of passing them directly to the `DataFrame` constructor? 

Currently there is this test case in io/tests/test_json/test_pandas.py that shadows a slightly confusing error message:

``` python
        # bad key
        json = StringIO('{"badkey":["A","B"],'
                        '"index":["2","3"],'
                        '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}')
        self.assertRaises(TypeError, read_json, json,
                          orient="split")
```

If you change the `TypeError` to a `ValueError` so the test fails, you get the following traceback:

```
ERROR: pandas.io.tests.test_json.test_pandas:TestPandasContainer.test_frame_from_json_bad_data
  /usr/local/bin/vim +256 pandas/io/tests/test_json/test_pandas.py  # test_frame_from_json_bad_data
    orient="split")
  /usr/local/bin/vim +471 python2.7/unittest/case.py  # assertRaises
    callableObj(*args, **kwargs)
  /usr/local/bin/vim +178 pandas/io/json.py  # read_json
    date_unit).parse()
  /usr/local/bin/vim +238 pandas/io/json.py  # parse
    self._parse_no_numpy()
  /usr/local/bin/vim +456 pandas/io/json.py  # _parse_no_numpy
    self.obj = DataFrame(dtype=None, **decoded)
TypeError: __init__() got an unexpected keyword argument 'badkey'
```



Discussion:

Resolving patch:
diff --git a/pandas/io/json.py b/pandas/io/json.py
--- a/pandas/io/json.py
+++ b/pandas/io/json.py
@@ -5,15 +5,14 @@
 
 import pandas.json as _json
 from pandas.tslib import iNaT
-from pandas.compat import long
+from pandas.compat import long, u
 from pandas import compat, isnull
 from pandas import Series, DataFrame, to_datetime
 from pandas.io.common import get_filepath_or_buffer
+import pandas.core.common as com
 
 loads = _json.loads
 dumps = _json.dumps
-
-
 ### interface to/from ###
 
 
@@ -230,6 +229,14 @@ def __init__(self, json, orient, dtype=True, convert_axes=True,
         self.keep_default_dates = keep_default_dates
         self.obj = None
 
+    def check_keys_split(self, decoded):
+        "checks that dict has only the appropriate keys for orient='split'"
+        bad_keys = set(decoded.keys()).difference(set(self._split_keys))
+        if bad_keys:
+            bad_keys = ", ".join(bad_keys)
+            raise ValueError(u("JSON data had unexpected key(s): %s") %
+                             com.pprint_thing(bad_keys))
+
     def parse(self):
 
         # try numpy
@@ -375,6 +382,8 @@ def _try_convert_dates(self):
 
 class SeriesParser(Parser):
     _default_orient = 'index'
+    _split_keys = ('name', 'index', 'data')
+
 
     def _parse_no_numpy(self):
 
@@ -385,6 +394,7 @@ def _parse_no_numpy(self):
                            for k, v in compat.iteritems(loads(
                                json,
                                precise_float=self.precise_float)))
+            self.check_keys_split(decoded)
             self.obj = Series(dtype=None, **decoded)
         else:
             self.obj = Series(
@@ -398,6 +408,7 @@ def _parse_numpy(self):
             decoded = loads(json, dtype=None, numpy=True,
                             precise_float=self.precise_float)
             decoded = dict((str(k), v) for k, v in compat.iteritems(decoded))
+            self.check_keys_split(decoded)
             self.obj = Series(**decoded)
         elif orient == "columns" or orient == "index":
             self.obj = Series(*loads(json, dtype=None, numpy=True,
@@ -418,6 +429,7 @@ def _try_convert_types(self):
 
 class FrameParser(Parser):
     _default_orient = 'columns'
+    _split_keys = ('columns', 'index', 'data')
 
     def _parse_numpy(self):
 
@@ -434,6 +446,7 @@ def _parse_numpy(self):
             decoded = loads(json, dtype=None, numpy=True,
                             precise_float=self.precise_float)
             decoded = dict((str(k), v) for k, v in compat.iteritems(decoded))
+            self.check_keys_split(decoded)
             self.obj = DataFrame(**decoded)
         elif orient == "values":
             self.obj = DataFrame(loads(json, dtype=None, numpy=True,
@@ -456,6 +469,7 @@ def _parse_no_numpy(self):
                            for k, v in compat.iteritems(loads(
                                json,
                                precise_float=self.precise_float)))
+            self.check_keys_split(decoded)
             self.obj = DataFrame(dtype=None, **decoded)
         elif orient == "index":
             self.obj = DataFrame(


Test patch:
diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py
--- a/pandas/io/tests/test_json/test_pandas.py
+++ b/pandas/io/tests/test_json/test_pandas.py
@@ -252,8 +252,8 @@ def test_frame_from_json_bad_data(self):
         json = StringIO('{"badkey":["A","B"],'
                         '"index":["2","3"],'
                         '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}')
-        self.assertRaises(TypeError, read_json, json,
-                          orient="split")
+        with tm.assertRaisesRegexp(ValueError, r"unexpected key\(s\): badkey"):
+            read_json(json, orient="split")
 
     def test_frame_from_json_nones(self):
         df = DataFrame([[1, 2], [4, 5, 6]])