Repository: pandas-dev/pandas

Issue:
Handling of nested JSON records
```
Is there a simple way of grabbing nested keys when constructing a Pandas Dataframe from JSON. Using the example JSON from below, how would I build a Dataframe that uses this column_header = ['id_str', 'text', 'user.screen_name'], (i.e. how do I get the 'screen_name' from the 'user' key without flattening the JSON).

Thanks,

Vishal

{   u'_id': ObjectId('4f8b95e8a504d022e2000000'),
    u'contributors': None,
    u'coordinates': None,
    u'created_at': u'Mon Apr 16 03:45:44 +0000 2012',
    u'entities': {   u'hashtags': [],
                     u'urls': [   {   u'display_url': u'stks.co/3M4X',
                                      u'expanded_url': u'http://stks.co/3M4X',
                                      u'indices': [72, 92],
                                      u'url': u'http://t.co/mGdTYusF'}],
                     u'user_mentions': []},
    u'favorited': False,
    u'geo': None,
    u'id': 191734090783916032L,
    u'id_str': u'191734090783916032',
    u'in_reply_to_screen_name': None,
    u'in_reply_to_status_id': None,
    u'in_reply_to_status_id_str': None,
    u'in_reply_to_user_id': None,
    u'in_reply_to_user_id_str': None,
    u'place': None,
    u'possibly_sensitive': False,
    u'possibly_sensitive_editable': True,
    u'processed_metadata': {   u'created_date': datetime.datetime(2012, 4, 16, 3, 45, 44, tzinfo=<bson.tz_util.FixedOffset object at 0x104d63790>),
                               u'search_queries': [   u'$AAPL',
                                                      u'$MSFT',
                                                      u'$GOOG'],
                               u'source': u'Twitter Streaming API'},
    u'retweet_count': 0,
    u'retweeted': False,
    u'source': u'<a href="http://stocktwits.com" rel="nofollow">StockTwits Web</a>',
    u'text': u'Interesting infographic on the internet and evolution of social media \u2794 http://t.co/mGdTYusF $FB $GOOG $TWIT $LNKD $AOL',
    u'truncated': False,
    u'user': {   u'_id': u'speculatethemkt',
                 u'contributors_enabled': False,
                 u'created_at': u'Tue Nov 30 02:28:20 +0000 2010',
                 u'default_profile': False,
                 u'default_profile_image': False,
                 u'description': u"I'm a 22-year old full-time forex trader living the location independent lifestyle. Author of The Trading Elite \u2794 http://amzn.to/I0nacY",
                 u'favourites_count': 1,
                 u'follow_request_sent': None,
                 u'followers_count': 19658,
                 u'following': None,
                 u'friends_count': 596,
                 u'geo_enabled': False,
                 u'id': 221226895,
                 u'id_str': u'221226895',
                 u'is_translator': False,
                 u'lang': u'en',
                 u'listed_count': 6,
                 u'location': u'Portland, OR',
                 u'name': u'Jared M.',
                 u'notifications': None,
                 u'processed_metadata': {   u'created_date': datetime.datetime(2012, 4, 16, 3, 45, 44, tzinfo=<bson.tz_util.FixedOffset object at 0x104d63790>),
                                            u'search_queries': [   u'$AAPL',
                                                                   u'$MSFT',
                                                                   u'$GOOG'],
                                            u'source': u'Twitter Streaming API'},
                 u'profile_background_color': u'4f4f4f',
                 u'profile_background_image_url': u'http://a0.twimg.com/profile_background_images/438353849/TWITTER-BG.jpg',
                 u'profile_background_image_url_https': u'https://si0.twimg.com/profile_background_images/438353849/TWITTER-BG.jpg',
                 u'profile_background_tile': False,
                 u'profile_image_url': u'http://a0.twimg.com/profile_images/2039368182/twitpic1_normal.png',
                 u'profile_image_url_https': u'https://si0.twimg.com/profile_images/2039368182/twitpic1_normal.png',
                 u'profile_link_color': u'bd0000',
                 u'profile_sidebar_border_color': u'eeeeee',
                 u'profile_sidebar_fill_color': u'efefef',
                 u'profile_text_color': u'333333',
                 u'profile_use_background_image': True,
                 u'protected': False,
                 u'screen_name': u'speculatethemkt',
                 u'show_all_inline_media': True,
                 u'statuses_count': 492,
                 u'time_zone': u'Pacific Time (US & Canada)',
                 u'url': u'http://www.speculatethemarkets.com',
                 u'utc_offset': -28800,
                 u'verified': False}}
```



Discussion:
related: http://stackoverflow.com/questions/16444797/which-is-the-most-efficient-way-of-loading-a-json-dataset-into-pandas-dataframes

this is invalid JSON (according to jsonlint), but generalized inference is IMHO too complicated, but #3804 should be able to do some of this. close this issue?

Is it feasible to grab the user section, (actually [this example from the other thread](https://github.com/pydata/pandas/pull/3804#issuecomment-19170850) is better):

Convert the `(data, posts)` section to DataFrame

```
s = r'''{
    "status": "success",
    "data": {
        "posts": [
            {
                "id": 1,
                "title": "A blog post",
                "body": "Some useful content"
            },
            {
                "id": 2,
                "title": "Another blog post",
                "body": "More content"
            }
        ]
    }
}'''

read_json(s, grab_nest=(data, posts)) # some better argument name
                  body  id              title
0  Some useful content   1        A blog post
1         More content   2  Another blog post
```

This is probably getting too cute.....

```
In [52]: def extract(df, l):
   ....:     for e in l:
   ....:         df = df[e]
   ....:     return df
   ....: 

In [54]: DataFrame.extract = extract

In [56]: DataFrame(pd.read_json(s).extract(['data','posts']))
Out[56]: 
                  body  id              title
0  Some useful content   1        A blog post
1         More content   2  Another blog post
```

Ha! Perhaps less overhead to do `pd.DataFrame(extract(pd.io.json.loads(s), ('data', 'posts')))`, but in either case we would lose the datetime parsing atm.

This could be a reasonably ok solution... tricky with orient (?), then parse_dates or whatever?

...other choice is just to loads/dumps/parse? :s

here's a bigish nested json: https://github.com/hayd/sf-city-lots-json ~200mb

I think I'd want to extract `['features']`.

I'm on an incredibly old macbook air, hence slow timings:

```
In [9]: %time with open('citylots.json', 'r') as f: pd.read_json(pd.io.json.dumps(extract(pd.io.json.loads(f.read()), ['features'])))
CPU times: user 29.13 s, sys: 28.27 s, total: 57.40 s
Wall time: 304.71 s

In [10]: %time with open('citylots.json', 'r') as f: pd.DataFrame(extract(pd.io.json.loads(f.read()), ['features']))
CPU times: user 11.96 s, sys: 11.79 s, total: 23.75 s
Wall time: 136.50 s

In [11]: %time with open('citylots.json', 'r') as f: pd.read_json(f.read())CPU times: user 13.47 s, sys: 10.41 s, total: 23.88 s
Wall time: 77.47 s
```

What is an extreme for reading in json?

After I figured out all I needed to do was clone the repository!
 (these also include full dtype conversions) FYI

```
In [3]: %time with open('citylots.json', 'r') as f: pd.read_json(pd.io.json.dumps(extract(pd.io.json.loads(f.read()), ['features'])))
CPU times: user 13.03 s, sys: 0.50 s, total: 13.53 s
Wall time: 15.12 s

In [6]: %time with open('citylots.json', 'r') as f: pd.DataFrame(extract(pd.io.json.loads(f.read()), ['features']))
CPU times: user 6.03 s, sys: 0.08 s, total: 6.11 s
Wall time: 6.13 s

In [7]: %time with open('citylots.json', 'r') as f: pd.read_json(f.read())
CPU times: user 6.27 s, sys: 0.16 s, total: 6.44 s
Wall time: 6.45 s
```

see #3876

Wow, I should never do any data analysis on that laptop... (sorry I forgot that you had to clone it).

But what I mean is, you'd lose the control from the read_json arguments. Will be interesting to see if this use case comes up a lot "in the wild".

... _really_ this really this kind of stuff should be done the other end, e.g. with http://www.elasticsearch.org/guide/reference/api/get/ (getting the _source directly).

I have a JSON normalization function I can clean up and make a PR before before anyone goes crazy writing one to save you some time. It would be nice to have a higher performance one at some point though

Hey @wesm and @hayd! I've been keeping my eye on this thread for a few days now—really impressive all the work that built up to this, thank you. Anyways, I though you might know that I _could_ go crazy writing some JSON normalization soon. Perhaps I should wait? :) 

Thanks for all.

Related: https://groups.google.com/forum/#!topic/pydata/XkiWtZKT698 (json is a list of nested dictionaries...)

Hey @nehalecky , I think @wesm says he has something in the works already, so perhaps if you can hold off til he's pushed, then you could hack that? :)


Resolving patch:
diff --git a/pandas/io/json.py b/pandas/io/json.py
--- a/pandas/io/json.py
+++ b/pandas/io/json.py
@@ -1,6 +1,8 @@
 # pylint: disable-msg=E1101,W0613,W0603
-import os
 
+import os
+import copy
+from collections import defaultdict
 import numpy as np
 
 import pandas.json as _json
@@ -15,7 +17,6 @@
 dumps = _json.dumps
 ### interface to/from ###
 
-
 def to_json(path_or_buf, obj, orient=None, date_format='epoch',
             double_precision=10, force_ascii=True, date_unit='ms'):
 
@@ -71,7 +72,6 @@ def write(self):
             date_unit=self.date_unit,
             iso_dates=self.date_format == 'iso')
 
-
 class SeriesWriter(Writer):
     _default_orient = 'index'
 
@@ -537,3 +537,201 @@ def is_ok(col):
             lambda col, c: self._try_convert_to_date(c),
             lambda col, c: ((self.keep_default_dates and is_ok(col))
                             or col in convert_dates))
+
+
+#----------------------------------------------------------------------
+# JSON normalization routines
+
+def nested_to_record(ds,prefix="",level=0):
+    """a simplified json_normalize
+
+    converts a nested dict into a flat dict ("record"), unlike json_normalize,
+    it does not attempt to extract a subset of the data.
+
+    Parameters
+    ----------
+    ds : dict or list of dicts
+
+    Returns
+    -------
+    d - dict or list of dicts, matching `ds`
+
+    Example:
+    IN[52]: nested_to_record(dict(flat1=1,dict1=dict(c=1,d=2),nested=dict(e=dict(c=1,d=2),d=2)))
+    Out[52]:
+    {'dict1.c': 1,
+     'dict1.d': 2,
+     'flat1': 1,
+     'nested.d': 2,
+     'nested.e.c': 1,
+     'nested.e.d': 2}
+    """
+    singleton = False
+    if isinstance(ds,dict):
+        ds = [ds]
+        singleton = True
+
+    new_ds = []
+    for d in ds:
+
+        new_d = copy.deepcopy(d)
+        for k,v in d.items():
+            # each key gets renamed with prefix
+            if level == 0:
+                newkey = str(k)
+            else:
+                newkey = prefix+'.'+ str(k)
+
+            # only dicts gets recurse-flattend
+            # only at level>1 do we rename the rest of the keys
+            if not isinstance(v,dict):
+                if level!=0: # so we skip copying for top level, common case
+                    v = new_d.pop(k)
+                    new_d[newkey]= v
+                continue
+            else:
+                v = new_d.pop(k)
+                new_d.update(nested_to_record(v,newkey,level+1))
+        new_ds.append(new_d)
+
+    if singleton:
+        return new_ds[0]
+    return new_ds
+
+
+def json_normalize(data, record_path=None, meta=None,
+                   meta_prefix=None,
+                   record_prefix=None):
+    """
+    "Normalize" semi-structured JSON data into a flat table
+
+    Parameters
+    ----------
+    data : dict or list of dicts
+        Unserialized JSON objects
+    record_path : string or list of strings, default None
+        Path in each object to list of records. If not passed, data will be
+        assumed to be an array of records
+    meta : list of paths (string or list of strings)
+        Fields to use as metadata for each record in resulting table
+    record_prefix : string, default None
+        If True, prefix records with dotted (?) path, e.g. foo.bar.field if
+        path to records is ['foo', 'bar']
+    meta_prefix : string, default None
+
+    Examples
+    --------
+    data = [{'state': 'Florida',
+             'shortname': 'FL',
+             'info': {
+                  'governor': 'Rick Scott'
+             },
+             'counties': [{'name': 'Dade', 'population': 12345},
+                         {'name': 'Broward', 'population': 40000},
+                         {'name': 'Palm Beach', 'population': 60000}]},
+            {'state': 'Ohio',
+             'shortname': 'OH',
+             'info': {
+                  'governor': 'John Kasich'
+             },
+             'counties': [{'name': 'Summit', 'population': 1234},
+                          {'name': 'Cuyahoga', 'population': 1337}]}]
+
+    result = json_normalize(data, 'counties', ['state', 'shortname',
+                                              ['info', 'governor']])
+
+      state    governor
+    Florida  Rick Scott
+
+
+    Returns
+    -------
+    frame : DataFrame
+    """
+    def _pull_field(js, spec):
+        result = js
+        if isinstance(spec, list):
+            for field in spec:
+                result = result[field]
+        else:
+            result = result[spec]
+
+        return result
+
+    # A bit of a hackjob
+    if isinstance(data, dict):
+        data = [data]
+
+    if record_path is None:
+        if any([isinstance(x,dict) for x in compat.itervalues(data[0])]):
+            # naive normalization, this is idempotent for flat records
+            # and potentially will inflate the data considerably for
+            # deeply nested structures:
+            #  {VeryLong: { b: 1,c:2}} -> {VeryLong.b:1 ,VeryLong.c:@}
+            #
+            # TODO: handle record value which are lists, at least error reasonabley
+            data = nested_to_record(data)
+        return DataFrame(data)
+    elif not isinstance(record_path, list):
+        record_path = [record_path]
+
+    if meta is None:
+        meta = []
+    elif not isinstance(meta, list):
+        meta = [meta]
+
+    for i, x in enumerate(meta):
+        if not isinstance(x, list):
+            meta[i] = [x]
+
+    # Disastrously inefficient for now
+    records = []
+    lengths = []
+
+    meta_vals = defaultdict(list)
+    meta_keys = ['.'.join(val) for val in meta]
+
+    def _recursive_extract(data, path, seen_meta, level=0):
+        if len(path) > 1:
+            for obj in data:
+                for val, key in zip(meta, meta_keys):
+                    if level + 1 == len(val):
+                        seen_meta[key] = _pull_field(obj, val[-1])
+
+                _recursive_extract(obj[path[0]], path[1:],
+                                   seen_meta, level=level+1)
+        else:
+            for obj in data:
+                recs = _pull_field(obj, path[0])
+
+                # For repeating the metadata later
+                lengths.append(len(recs))
+
+                for val, key in zip(meta, meta_keys):
+                    if level + 1 > len(val):
+                        meta_val = seen_meta[key]
+                    else:
+                        meta_val = _pull_field(obj, val[level:])
+                    meta_vals[key].append(meta_val)
+
+                records.extend(recs)
+
+    _recursive_extract(data, record_path, {}, level=0)
+
+    result = DataFrame(records)
+
+    if record_prefix is not None:
+        result.rename(columns=lambda x: record_prefix + x, inplace=True)
+
+    # Data types, a problem
+    for k, v in compat.iteritems(meta_vals):
+        if meta_prefix is not None:
+            k = meta_prefix + k
+
+        if k in result:
+            raise ValueError('Conflicting metadata name %s, '
+                             'need distinguishing prefix ' % k)
+
+        result[k] = np.array(v).repeat(lengths)
+
+    return result


Test patch:
diff --git a/pandas/io/tests/test_json_norm.py b/pandas/io/tests/test_json_norm.py
new file mode 100644
--- /dev/null
+++ b/pandas/io/tests/test_json_norm.py
@@ -0,0 +1,208 @@
+import nose
+import unittest
+
+from pandas import DataFrame
+import numpy as np
+
+import pandas.util.testing as tm
+
+from pandas.io.json import json_normalize, nested_to_record
+
+def _assert_equal_data(left, right):
+    if not left.columns.equals(right.columns):
+        left = left.reindex(columns=right.columns)
+
+    tm.assert_frame_equal(left, right)
+
+
+class TestJSONNormalize(unittest.TestCase):
+
+    def setUp(self):
+        self.state_data = [
+             {'counties': [{'name': 'Dade', 'population': 12345},
+                           {'name': 'Broward', 'population': 40000},
+                           {'name': 'Palm Beach', 'population': 60000}],
+              'info': {'governor': 'Rick Scott'},
+              'shortname': 'FL',
+              'state': 'Florida'},
+             {'counties': [{'name': 'Summit', 'population': 1234},
+                           {'name': 'Cuyahoga', 'population': 1337}],
+              'info': {'governor': 'John Kasich'},
+              'shortname': 'OH',
+              'state': 'Ohio'}]
+
+    def test_simple_records(self):
+        recs = [{'a': 1, 'b': 2, 'c': 3},
+                {'a': 4, 'b': 5, 'c': 6},
+                {'a': 7, 'b': 8, 'c': 9},
+                {'a': 10, 'b': 11, 'c': 12}]
+
+        result = json_normalize(recs)
+        expected = DataFrame(recs)
+
+        tm.assert_frame_equal(result, expected)
+
+    def test_simple_normalize(self):
+        result = json_normalize(self.state_data[0], 'counties')
+        expected = DataFrame(self.state_data[0]['counties'])
+        tm.assert_frame_equal(result, expected)
+
+        result = json_normalize(self.state_data, 'counties')
+
+        expected = []
+        for rec in self.state_data:
+            expected.extend(rec['counties'])
+        expected = DataFrame(expected)
+
+        tm.assert_frame_equal(result, expected)
+
+        result = json_normalize(self.state_data, 'counties', meta='state')
+        expected['state'] = np.array(['Florida', 'Ohio']).repeat([3, 2])
+
+        tm.assert_frame_equal(result, expected)
+
+    def test_more_deeply_nested(self):
+        data = [{'country': 'USA',
+                 'states': [{'name': 'California',
+                             'cities': [{'name': 'San Francisco',
+                                         'pop': 12345},
+                                        {'name': 'Los Angeles',
+                                         'pop': 12346}]
+                            },
+                            {'name': 'Ohio',
+                             'cities': [{'name': 'Columbus',
+                                         'pop': 1234},
+                                        {'name': 'Cleveland',
+                                         'pop': 1236}]}
+                           ]
+                 },
+                {'country': 'Germany',
+                 'states': [{'name': 'Bayern',
+                             'cities': [{'name': 'Munich', 'pop': 12347}]
+                            },
+                            {'name': 'Nordrhein-Westfalen',
+                             'cities': [{'name': 'Duesseldorf', 'pop': 1238},
+                                        {'name': 'Koeln', 'pop': 1239}]}
+                           ]
+                 }
+                ]
+
+        result = json_normalize(data, ['states', 'cities'],
+                                meta=['country', ['states', 'name']])
+                                # meta_prefix={'states': 'state_'})
+
+        ex_data = {'country': ['USA'] * 4 + ['Germany'] * 3,
+                   'states.name': ['California', 'California', 'Ohio', 'Ohio',
+                                   'Bayern', 'Nordrhein-Westfalen',
+                                   'Nordrhein-Westfalen'],
+                   'name': ['San Francisco', 'Los Angeles', 'Columbus',
+                            'Cleveland', 'Munich', 'Duesseldorf', 'Koeln'],
+                   'pop': [12345, 12346, 1234, 1236, 12347, 1238, 1239]}
+
+        expected = DataFrame(ex_data, columns=result.columns)
+        tm.assert_frame_equal(result, expected)
+
+    def test_shallow_nested(self):
+        data = [{'state': 'Florida',
+                 'shortname': 'FL',
+                 'info': {
+                      'governor': 'Rick Scott'
+                 },
+                 'counties': [{'name': 'Dade', 'population': 12345},
+                             {'name': 'Broward', 'population': 40000},
+                             {'name': 'Palm Beach', 'population': 60000}]},
+                {'state': 'Ohio',
+                 'shortname': 'OH',
+                 'info': {
+                      'governor': 'John Kasich'
+                 },
+                 'counties': [{'name': 'Summit', 'population': 1234},
+                              {'name': 'Cuyahoga', 'population': 1337}]}]
+
+        result = json_normalize(data, 'counties',
+                                ['state', 'shortname',
+                                 ['info', 'governor']])
+        ex_data = {'name': ['Dade', 'Broward', 'Palm Beach', 'Summit',
+                            'Cuyahoga'],
+                   'state': ['Florida'] * 3 + ['Ohio'] * 2,
+                   'shortname': ['FL', 'FL', 'FL', 'OH', 'OH'],
+                   'info.governor': ['Rick Scott'] * 3 + ['John Kasich'] * 2,
+                   'population': [12345, 40000, 60000, 1234, 1337]}
+        expected = DataFrame(ex_data, columns=result.columns)
+        tm.assert_frame_equal(result, expected)
+
+    def test_meta_name_conflict(self):
+        data = [{'foo': 'hello',
+                 'bar': 'there',
+                 'data': [{'foo': 'something', 'bar': 'else'},
+                          {'foo': 'something2', 'bar': 'else2'}]}]
+
+        self.assertRaises(ValueError, json_normalize, data,
+                          'data', meta=['foo', 'bar'])
+
+        result = json_normalize(data, 'data', meta=['foo', 'bar'],
+                                meta_prefix='meta')
+
+        for val in ['metafoo', 'metabar', 'foo', 'bar']:
+            self.assertTrue(val in result)
+
+    def test_record_prefix(self):
+        result = json_normalize(self.state_data[0], 'counties')
+        expected = DataFrame(self.state_data[0]['counties'])
+        tm.assert_frame_equal(result, expected)
+
+        result = json_normalize(self.state_data, 'counties',
+                                meta='state',
+                                record_prefix='county_')
+
+        expected = []
+        for rec in self.state_data:
+            expected.extend(rec['counties'])
+        expected = DataFrame(expected)
+        expected = expected.rename(columns=lambda x: 'county_' + x)
+        expected['state'] = np.array(['Florida', 'Ohio']).repeat([3, 2])
+
+        tm.assert_frame_equal(result, expected)
+
+
+class TestNestedToRecord(unittest.TestCase):
+
+    def test_flat_stays_flat(self):
+        recs = [dict(flat1=1,flat2=2),
+                dict(flat1=3,flat2=4),
+                ]
+
+        result = nested_to_record(recs)
+        expected = recs
+        self.assertEqual(result, expected)
+
+    def test_one_level_deep_flattens(self):
+        data = dict(flat1=1,
+                    dict1=dict(c=1,d=2))
+
+        result = nested_to_record(data)
+        expected =     {'dict1.c': 1,
+             'dict1.d': 2,
+             'flat1': 1}
+
+        self.assertEqual(result,expected)
+
+    def test_nested_flattens(self):
+        data = dict(flat1=1,
+                    dict1=dict(c=1,d=2),
+                    nested=dict(e=dict(c=1,d=2),
+                                d=2))
+
+        result = nested_to_record(data)
+        expected =     {'dict1.c': 1,
+             'dict1.d': 2,
+             'flat1': 1,
+             'nested.d': 2,
+             'nested.e.c': 1,
+             'nested.e.d': 2}
+
+        self.assertEqual(result,expected)
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb',
+                         '--pdb-failure', '-s'], exit=False)
diff --git a/test.py b/test.py
new file mode 100644
--- /dev/null
+++ b/test.py
@@ -0,0 +1,12 @@
+
+
+import pandas as pd
+df = pd.DataFrame(
+    {'pid' : [1,1,1,2,2,3,3,3],
+     'tag' : [23,45,62,24,45,34,25,62],
+          })
+
+g = df.groupby('tag')
+
+import pdb; pdb.set_trace()
+g.filter(lambda x: len(x) > 1)