Repository: pandas-dev/pandas

Issue:
BUG: validate multi index names in HDFStore writing
```
store_id_map
```

> <class 'pandas.io.pytables.HDFStore'>
> File path: C:\output\identifier_map.h5
> /identifier_map            frame_table  (typ->appendable_multi,nrows->26779823,ncols->9,indexers->[index],dc->[RefIdentifierID])

```
store_id_map.select('identifier_map')
```

> ---------------------------------------------------------------------------
> KeyError                                  Traceback (most recent call last)
> <ipython-input-21-4b0da9c6ab3c> in <module>()
> ----> 1 store_id_map.select('identifier_map')
> 
> C:\Python27\lib\site-packages\pandas\io\pytables.pyc in select(self, key, where, start, stop, columns, iterator, chunksize, auto_close, *_kwargs)
>     456             return TableIterator(self, func, nrows=s.nrows, start=start, stop=stop, chunksize=chunksize, auto_close=auto_close)
>     457 
> --> 458         return TableIterator(self, func, nrows=s.nrows, start=start, stop=stop, auto_close=auto_close).get_values()
>     459 
>     460     def select_as_coordinates(self, key, where=None, start=None, stop=None, *_kwargs):
> 
> C:\Python27\lib\site-packages\pandas\io\pytables.pyc in get_values(self)
>     982 
>     983     def get_values(self):
> --> 984         results = self.func(self.start, self.stop)
>     985         self.close()
>     986         return results
> 
> C:\Python27\lib\site-packages\pandas\io\pytables.pyc in func(_start, _stop)
>     449         # what we are actually going to do for a chunk
>     450         def func(_start, _stop):
> --> 451             return s.read(where=where, start=_start, stop=_stop, columns=columns, **kwargs)
>     452 
>     453         if iterator or chunksize is not None:
> 
> C:\Python27\lib\site-packages\pandas\io\pytables.pyc in read(self, columns, *_kwargs)
>    3259                     columns.insert(0, n)
>    3260         df = super(AppendableMultiFrameTable, self).read(columns=columns, *_kwargs)
> -> 3261         df.set_index(self.levels, inplace=True)
>    3262         return df
>    3263 
> 
> C:\Python27\lib\site-packages\pandas\core\frame.pyc in set_index(self, keys, drop, append, inplace, verify_integrity)
>    2827                 names.append(None)
>    2828             else:
> -> 2829                 level = frame[col].values
>    2830                 names.append(col)
>    2831                 if drop:
> 
> C:\Python27\lib\site-packages\pandas\core\frame.pyc in **getitem**(self, key)
>    2001             # get column
>    2002             if self.columns.is_unique:
> -> 2003                 return self._get_item_cache(key)
>    2004 
>    2005             # duplicate columns
> 
> C:\Python27\lib\site-packages\pandas\core\generic.pyc in _get_item_cache(self, item)
>     665             return cache[item]
>     666         except Exception:
> --> 667             values = self._data.get(item)
>     668             res = self._box_item_values(item, values)
>     669             cache[item] = res
> 
> C:\Python27\lib\site-packages\pandas\core\internals.pyc in get(self, item)
>    1653     def get(self, item):
>    1654         if self.items.is_unique:
> -> 1655             _, block = self._find_block(item)
>    1656             return block.get(item)
>    1657         else:
> 
> C:\Python27\lib\site-packages\pandas\core\internals.pyc in _find_block(self, item)
>    1933 
>    1934     def _find_block(self, item):
> -> 1935         self._check_have(item)
>    1936         for i, block in enumerate(self.blocks):
>    1937             if item in block:
> 
> C:\Python27\lib\site-packages\pandas\core\internals.pyc in _check_have(self, item)
>    1940     def _check_have(self, item):
>    1941         if item not in self.items:
> -> 1942             raise KeyError('no item named %s' % com.pprint_thing(item))
>    1943 
>    1944     def reindex_axis(self, new_axis, method=None, axis=0, copy=True):
> 
> KeyError: u'no item named None'

I am at a loss. What does that mean? I successfully saved the DataFrame but I cannot read it back.



Discussion:
Never mind. One of the columns of the MultiIndex was not named, i.e. it had None as a name. Apparently HDFStore chokes on that. If so, you guys might want to raise an exception when we try to save a frame like that.

That's definitely a bug there, so we should address that.


Resolving patch:
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -2688,6 +2688,14 @@ def is_multi_index(self):
         """ the levels attribute is 1 or a list in the case of a multi-index """
         return isinstance(self.levels,list)
 
+    def validate_multiindex(self, obj):
+        """ validate that we can store the multi-index; reset and return the new object """
+        levels = [ l if l is not None else "level_{0}".format(i) for i, l in enumerate(obj.index.names) ]
+        try:
+            return obj.reset_index(), levels
+        except (ValueError):
+            raise ValueError("duplicate names/columns in the multi-index when storing as a table")
+
     @property
     def nrows_expected(self):
         """ based on our axes, compute the expected nrows """
@@ -3701,10 +3709,9 @@ class AppendableMultiSeriesTable(AppendableSeriesTable):
     def write(self, obj, **kwargs):
         """ we are going to write this as a frame table """
         name = obj.name or 'values'
-        cols = list(obj.index.names)
+        obj, self.levels = self.validate_multiindex(obj)
+        cols = list(self.levels)
         cols.append(name)
-        self.levels = list(obj.index.names)
-        obj = obj.reset_index()
         obj.columns = cols
         return super(AppendableMultiSeriesTable, self).write(obj=obj, **kwargs)
 
@@ -3764,6 +3771,7 @@ class AppendableMultiFrameTable(AppendableFrameTable):
     table_type = u('appendable_multiframe')
     obj_type = DataFrame
     ndim = 2
+    _re_levels = re.compile("^level_\d+$")
 
     @property
     def table_type_short(self):
@@ -3774,11 +3782,11 @@ def write(self, obj, data_columns=None, **kwargs):
             data_columns = []
         elif data_columns is True:
             data_columns = obj.columns[:]
-        for n in obj.index.names:
+        obj, self.levels = self.validate_multiindex(obj)
+        for n in self.levels:
             if n not in data_columns:
                 data_columns.insert(0, n)
-        self.levels = obj.index.names
-        return super(AppendableMultiFrameTable, self).write(obj=obj.reset_index(), data_columns=data_columns, **kwargs)
+        return super(AppendableMultiFrameTable, self).write(obj=obj, data_columns=data_columns, **kwargs)
 
     def read(self, columns=None, **kwargs):
         if columns is not None:
@@ -3787,7 +3795,11 @@ def read(self, columns=None, **kwargs):
                     columns.insert(0, n)
         df = super(AppendableMultiFrameTable, self).read(
             columns=columns, **kwargs)
-        df.set_index(self.levels, inplace=True)
+        df = df.set_index(self.levels)
+
+        # remove names for 'level_%d'
+        df.index = df.index.set_names([ None if self._re_levels.search(l) else l for l in df.index.names ])
+
         return df
 
 


Test patch:
diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py
--- a/pandas/io/tests/test_pytables.py
+++ b/pandas/io/tests/test_pytables.py
@@ -1572,6 +1572,51 @@ def test_column_multiindex(self):
             store.put('df1',df,format='table')
             tm.assert_frame_equal(store['df1'],df,check_index_type=True,check_column_type=True)
 
+    def test_store_multiindex(self):
+
+        # validate multi-index names
+        # GH 5527
+        with ensure_clean_store(self.path) as store:
+
+            def make_index(names=None):
+                return MultiIndex.from_tuples([( datetime.datetime(2013,12,d), s, t) for d in range(1,3) for s in range(2) for t in range(3)],
+                                              names=names)
+
+
+            # no names
+            _maybe_remove(store, 'df')
+            df = DataFrame(np.zeros((12,2)), columns=['a','b'], index=make_index())
+            store.append('df',df)
+            tm.assert_frame_equal(store.select('df'),df)
+
+            # partial names
+            _maybe_remove(store, 'df')
+            df = DataFrame(np.zeros((12,2)), columns=['a','b'], index=make_index(['date',None,None]))
+            store.append('df',df)
+            tm.assert_frame_equal(store.select('df'),df)
+
+            # series
+            _maybe_remove(store, 's')
+            s = Series(np.zeros(12), index=make_index(['date',None,None]))
+            store.append('s',s)
+            tm.assert_series_equal(store.select('s'),s)
+
+            # dup with column
+            _maybe_remove(store, 'df')
+            df = DataFrame(np.zeros((12,2)), columns=['a','b'], index=make_index(['date','a','t']))
+            self.assertRaises(ValueError, store.append, 'df',df)
+
+            # dup within level
+            _maybe_remove(store, 'df')
+            df = DataFrame(np.zeros((12,2)), columns=['a','b'], index=make_index(['date','date','date']))
+            self.assertRaises(ValueError, store.append, 'df',df)
+
+            # fully names
+            _maybe_remove(store, 'df')
+            df = DataFrame(np.zeros((12,2)), columns=['a','b'], index=make_index(['date','s','t']))
+            store.append('df',df)
+            tm.assert_frame_equal(store.select('df'),df)
+
     def test_pass_spec_to_storer(self):
 
         df = tm.makeDataFrame()