Repository: pandas-dev/pandas

Issue:
BUG: read_json -> 'DataFrame' object has no attribute 'dtype'
Came across this when creating #4376. Not sure what's going on as I was under the impression that DataFrame always has a `dtype` attribute. 

The error only seems to occur when the columns are not unique:

``` python
In [5]: import pandas as pd

In [6]: df = pd.DataFrame([['a','b'],['c','d']], index=[1,2], columns=['x','y'])

In [7]: df
Out[7]: 
   x  y
1  a  b
2  c  d

In [8]: pd.read_json(df.to_json(orient='split'), orient='split')
Out[8]: 
   x  y
1  a  b
2  c  d

In [9]: df = pd.DataFrame([['a','b'],['c','d']], index=[1,2], columns=['x','x'])

In [10]: df
Out[10]: 
   x  x
1  a  b
2  c  d

In [11]: pd.read_json(df.to_json(orient='split'), orient='split')
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-11-aaa9a429e682> in <module>()
----> 1 pd.read_json(df.to_json(orient='split'), orient='split')

/home/kieran/work/git/pandas_kom/pandas/io/json.pyc in read_json(path_or_buf, orient, typ, dtype, convert_axes, convert_dates, keep_default_dates, numpy, precise_float)
    196     obj = None
    197     if typ == 'frame':
--> 198         obj = FrameParser(json, orient, dtype, convert_axes, convert_dates, keep_default_dates, numpy).parse()
    199 
    200     if typ == 'series' or obj is None:

/home/kieran/work/git/pandas_kom/pandas/io/json.pyc in parse(self)
    241         if self.convert_axes:
    242             self._convert_axes()
--> 243         self._try_convert_types()
    244         return self.obj
    245 

/home/kieran/work/git/pandas_kom/pandas/io/json.pyc in _try_convert_types(self)
    453             self._try_convert_dates()
    454         for col in self.obj.columns:
--> 455             new_data, result = self._try_convert_data(col, self.obj[col], convert_dates=False)
    456             if result:
    457                 self.obj[col] = new_data

/home/kieran/work/git/pandas_kom/pandas/io/json.pyc in _try_convert_data(self, name, data, use_dtypes, convert_dates)
    282         result = False
    283 
--> 284         if data.dtype == 'object':
    285 
    286             # try float

/home/kieran/work/git/pandas_kom/pandas/core/frame.pyc in __getattr__(self, name)
   2086             return self[name]
   2087         raise AttributeError("'%s' object has no attribute '%s'" %
-> 2088                              (type(self).__name__, name))
   2089 
   2090     def __setattr__(self, name, value):

AttributeError: 'DataFrame' object has no attribute 'dtype'

```



Discussion:
it is expecting the object to be a `series` at that point....let me look

actually this is a problem with iterating over the duplicate columns that are read back...easy fix


Resolving patch:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -1538,23 +1538,23 @@ def _interleave(self, items):
         # By construction, all of the item should be covered by one of the
         # blocks
         if items.is_unique:
+
             for block in self.blocks:
                 indexer = items.get_indexer(block.items)
                 if (indexer == -1).any():
                     raise AssertionError('Items must contain all block items')
                 result[indexer] = block.get_values(dtype)
                 itemmask[indexer] = 1
+
+            if not itemmask.all():
+                raise AssertionError('Some items were not contained in blocks')
+
         else:
-            for block in self.blocks:
-                mask = items.isin(block.items)
-                indexer = mask.nonzero()[0]
-                if (len(indexer) != len(block.items)):
-                    raise AssertionError('All items must be in block items')
-                result[indexer] = block.get_values(dtype)
-                itemmask[indexer] = 1
 
-        if not itemmask.all():
-            raise AssertionError('Some items were not contained in blocks')
+            # non-unique, must use ref_locs
+            rl = self._set_ref_locs()
+            for i, (block, idx) in enumerate(rl):
+                result[i] = block.iget(idx)
 
         return result
 
diff --git a/pandas/io/json.py b/pandas/io/json.py
--- a/pandas/io/json.py
+++ b/pandas/io/json.py
@@ -52,19 +52,24 @@ def __init__(self, obj, orient, date_format, double_precision, ensure_ascii):
         self._format_axes()
         self._format_dates()
 
+    def _needs_to_date(self, obj):
+        return obj.dtype == 'datetime64[ns]'
+
     def _format_dates(self):
         raise NotImplementedError
 
     def _format_axes(self):
         raise NotImplementedError
 
-    def _needs_to_date(self, data):
-        return self.date_format == 'iso' and data.dtype == 'datetime64[ns]'
-
     def _format_to_date(self, data):
-        if self._needs_to_date(data):
+
+        # iso
+        if self.date_format == 'iso':
             return data.apply(lambda x: x.isoformat())
-        return data
+
+        # int64
+        else:
+            return data.astype(np.int64)
 
     def copy_if_needed(self):
         """ copy myself if necessary """
@@ -87,13 +92,11 @@ def _format_axes(self):
             self.obj.index = self._format_to_date(self.obj.index.to_series())
 
     def _format_dates(self):
-        if self._needs_to_date(self.obj):
-            self.copy_if_needed()
+        if self.obj.dtype == 'datetime64[ns]':
             self.obj = self._format_to_date(self.obj)
 
     def _format_bools(self):
         if self._needs_to_bool(self.obj):
-            self.copy_if_needed()
             self.obj = self._format_to_bool(self.obj)
 
 class FrameWriter(Writer):
@@ -123,13 +126,22 @@ def _format_axes(self):
             setattr(self.obj,axis,self._format_to_date(a.to_series()))
 
     def _format_dates(self):
-        if self.date_format == 'iso':
-            dtypes = self.obj.dtypes
-            dtypes = dtypes[dtypes == 'datetime64[ns]']
-            if len(dtypes):
-                self.copy_if_needed()
-                for c in dtypes.index:
-                    self.obj[c] = self._format_to_date(self.obj[c])
+        dtypes = self.obj.dtypes
+        if len(dtypes[dtypes == 'datetime64[ns]']):
+
+            # need to create a new object
+            d = {}
+
+            for i, (col, c) in enumerate(self.obj.iteritems()):
+
+                if c.dtype == 'datetime64[ns]':
+                    c = self._format_to_date(c)
+
+                d[i] = c
+
+            d = DataFrame(d,index=self.obj.index)
+            d.columns = self.obj.columns
+            self.obj = d
 
 def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
               convert_axes=True, convert_dates=True, keep_default_dates=True,
@@ -291,14 +303,16 @@ def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True):
             except:
                 pass
 
-        if data.dtype == 'float':
+        if data.dtype.kind == 'f':
 
-            # coerce floats to 64
-            try:
-                data = data.astype('float64')
-                result = True
-            except:
-                pass
+            if data.dtype != 'float64':
+
+                # coerce floats to 64
+                try:
+                    data = data.astype('float64')
+                    result = True
+                except:
+                    pass
 
         # do't coerce 0-len data
         if len(data) and (data.dtype == 'float' or data.dtype == 'object'):
@@ -448,14 +462,35 @@ def _parse_no_numpy(self):
             self.obj = DataFrame(
                 loads(json, precise_float=self.precise_float), dtype=None)
 
+    def _process_converter(self, f, filt=None):
+        """ take a conversion function and possibly recreate the frame """
+
+        if filt is None:
+            filt = lambda col, c: True
+
+        needs_new_obj = False
+        new_obj = dict()
+        for i, (col, c) in enumerate(self.obj.iteritems()):
+            if filt(col, c):
+                new_data, result = f(col, c)
+                if result:
+                    c = new_data
+                    needs_new_obj = True
+            new_obj[i] = c
+
+        if needs_new_obj:
+
+            # possibly handle dup columns
+            new_obj = DataFrame(new_obj,index=self.obj.index)
+            new_obj.columns = self.obj.columns
+            self.obj = new_obj
+
     def _try_convert_types(self):
         if self.obj is None: return
         if self.convert_dates:
             self._try_convert_dates()
-        for col in self.obj.columns:
-            new_data, result = self._try_convert_data(col, self.obj[col], convert_dates=False)
-            if result:
-                self.obj[col] = new_data
+
+        self._process_converter(lambda col, c: self._try_convert_data(col, c, convert_dates=False))
 
     def _try_convert_dates(self):
         if self.obj is None: return
@@ -478,9 +513,6 @@ def is_ok(col):
                     return True
             return False
 
+        self._process_converter(lambda col, c: self._try_convert_to_date(c),
+                                lambda col, c: (self.keep_default_dates and is_ok(col)) or col in convert_dates)
 
-        for col in self.obj.columns:
-            if (self.keep_default_dates and is_ok(col)) or col in convert_dates:
-                new_data, result = self._try_convert_to_date(self.obj[col])
-                if result:
-                    self.obj[col] = new_data


Test patch:
diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py
--- a/pandas/io/tests/test_json/test_pandas.py
+++ b/pandas/io/tests/test_json/test_pandas.py
@@ -83,6 +83,21 @@ def test_frame_non_unique_columns(self):
         unser = read_json(df.to_json(orient='values'), orient='values')
         np.testing.assert_equal(df.values, unser.values)
 
+        # GH4377; duplicate columns not processing correctly
+        df = DataFrame([['a','b'],['c','d']], index=[1,2], columns=['x','y'])
+        result = read_json(df.to_json(orient='split'), orient='split')
+        assert_frame_equal(result, df)
+
+        def _check(df):
+            result = read_json(df.to_json(orient='split'), orient='split', convert_dates=['x'])
+            assert_frame_equal(result, df)
+
+        for o in [[['a','b'],['c','d']],
+                  [[1.5,2.5],[3.5,4.5]],
+                  [[1,2.5],[3,4.5]],
+                  [[Timestamp('20130101'),3.5],[Timestamp('20130102'),4.5]]]:
+            _check(DataFrame(o, index=[1,2], columns=['x','x']))
+
     def test_frame_from_json_to_json(self):
 
         def _check_orient(df, orient, dtype=None, numpy=False, convert_axes=True, check_dtype=True, raise_ok=None):
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -2950,6 +2950,12 @@ def check(result, expected=None):
         expected = DataFrame([[1],[1],[1]],columns=['bar'])
         check(df,expected)
 
+        # values
+        df = DataFrame([[1,2.5],[3,4.5]], index=[1,2], columns=['x','x'])
+        result = df.values
+        expected = np.array([[1,2.5],[3,4.5]])
+        self.assert_((result == expected).all().all())
+
     def test_insert_benchmark(self):
         # from the vb_suite/frame_methods/frame_insert_columns
         N = 10