Repository: pandas-dev/pandas

Issue:
msgpack unpack dataframe error: InvalidIndexError
This happen with some Pandas dataframes with indexes. 

``` python
from pandas import read_msgpack
msg = df.to_msgpack()
read_msgpack(msg) #==> exception!
```

Full Traceback (most recent call last):

File "/srv/python/venv/local/lib/python2.7/site-packages/pandas/io/packers.py", line 163, in read_msgpack
return read(fh)
File "/srv/python/venv/local/lib/python2.7/site-packages/pandas/io/packers.py", line 141, in read
l = list(unpack(fh))
File "pandas/msgpack.pyx", line 662, in pandas.msgpack.Unpacker.**next** (pandas/msgpack.cpp:8303)
File "pandas/msgpack.pyx", line 591, in pandas.msgpack.Unpacker._unpack (pandas/msgpack.cpp:7328)
File "/srv/python/venv/local/lib/python2.7/site-packages/pandas/io/packers.py", line 490, in decode
blocks = [create_block(b) for b in obj['blocks']]
File "/srv/python/venv/local/lib/python2.7/site-packages/pandas/io/packers.py", line 488, in create_block
placement=axes[0].get_indexer(b['items']))
File "/srv/python/venv/local/lib/python2.7/site-packages/pandas/core/index.py", line 1488, in get_indexer
raise InvalidIndexError('Reindexing only valid with uniquely'
InvalidIndexError: Reindexing only valid with uniquely valued Index objects



Discussion:
show `pd.show_versions()` a sample of the data frame and `df.index`, and `df.index.is_unique`

# pd.show_versions()

INSTALLED VERSIONS

commit: None
python: 2.7.8.final.0
python-bits: 64
OS: Linux
OS-release: 3.8.0-29-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: 0.15.0

# df.index

Int64Index([22, 29, 15, 11, 16, 43, 23, 30, 38, 24, 5, 9, 25, 7, 47, 4, 12, 26, 10, 17, 31, 39, 46, 6, 27, 18, 44, 41, 13, 8, 19, 20, 32, 52, 48, 0, 2, 3, 34, 35, 33, 49, 51, 1, 50], dtype='int64')

# df.index.is_unique

result_df.index.is_unique == True

# Sample dataframe

,d-37_id,d-37,m5010,m5010
22,29,201202,24330.47,24330.47
29,30,201203,56512.66,56512.66
15,31,201204,40812.07,40812.07
11,32,201205,115167.1,115167.1
16,33,201206,65513.84,65513.84
43,34,201207,20229.75,20229.75
23,35,201208,47008.03,47008.03
30,36,201209,69365.29,69365.29
38,37,201210,23771.42,23771.42
24,38,201211,61179.85,61179.85
5,39,201212,35658.55,35658.55
9,16,201301,162725.08,162725.08
25,17,201302,30242.13,30242.13
7,18,201303,100384.58,100384.58
47,19,201304,90338.54,90338.54
4,20,201305,75657.68,75657.68
12,21,201306,187771.41,187771.41
26,22,201307,70994.8,70994.8
10,23,201308,39255.37,39255.37
17,24,201309,232711.12,232711.12
31,25,201310,65818.43,65818.43
39,26,201311,184484.82,184484.82
46,27,201312,113374.1,113374.1
6,15,201401,129522.36,129522.36
27,40,201402,111226.45,111226.45
18,41,201403,64987.46,64987.46
44,42,201404,201649.92,201649.92
41,43,201405,44253.82,44253.82
13,44,201406,177624.59,177624.59
8,45,201407,51742.21,51742.21
19,62,201408,62129.06,62129.06
20,63,201409,196536.66,196536.66
32,64,201410,83913.08,83913.08
52,65,201411,6765.87,6765.87
48,66,201412,4585.0,4585.0
0,53,201501,282906.59,282906.59
2,54,201502,243472.38,243472.38
3,55,201503,150515.79,150515.79
34,56,201504,262266.21,262266.21
35,57,201505,57971.76,57971.76
33,58,201506,26562.02,26562.02
49,59,201507,53539.97,53539.97
51,70,201512,4231.5,4231.5
1,49,201601,120828.2,120828.2
50,71,201701,10448.92,10448.92

This works in 0.15.2

You can try to upgrade (I do recall this bug getting fixed, but can't find ATM).
or show an example which fails.

```
In [15]: pd.read_msgpack(df.to_msgpack())
Out[15]: 
    d-37_id    d-37      m5010    m5010.1
22       29  201202   24330.47   24330.47
29       30  201203   56512.66   56512.66
```

This does not seem to be fixed and happens when the column index is not unique:

```
pd.read_msgpack(pd.DataFrame(columns=["a", "a"]).to_msgpack())
```

I thought I had reported this error, will try to find it.


Resolving patch:
diff --git a/pandas/io/packers.py b/pandas/io/packers.py
--- a/pandas/io/packers.py
+++ b/pandas/io/packers.py
@@ -357,6 +357,7 @@ def encode(obj):
                     'klass': obj.__class__.__name__,
                     'axes': data.axes,
                     'blocks': [{'items': data.items.take(b.mgr_locs),
+                                'locs': b.mgr_locs.as_array,
                                 'values': convert(b.values),
                                 'shape': b.values.shape,
                                 'dtype': b.dtype.num,
@@ -485,9 +486,15 @@ def decode(obj):
         def create_block(b):
             values = unconvert(b['values'], dtype_for(b['dtype']),
                                b['compress']).reshape(b['shape'])
+
+            # locs handles duplicate column names, and should be used instead of items; see GH 9618
+            if 'locs' in b:
+                placement = b['locs']
+            else:
+                placement = axes[0].get_indexer(b['items'])
             return make_block(values=values,
                               klass=getattr(internals, b['klass']),
-                              placement=axes[0].get_indexer(b['items']))
+                              placement=placement)
 
         blocks = [create_block(b) for b in obj['blocks']]
         return globals()[obj['klass']](BlockManager(blocks, axes))
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -537,6 +537,7 @@ def pxd(name):
                 ],
       package_data={'pandas.io': ['tests/data/legacy_hdf/*.h5',
                                   'tests/data/legacy_pickle/*/*.pickle',
+                                  'tests/data/legacy_msgpack/*/*.msgpack',
                                   'tests/data/*.csv*',
                                   'tests/data/*.dta',
                                   'tests/data/*.txt',


Test patch:
diff --git a/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_2.7.10.msgpack b/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_2.7.10.msgpack
new file mode 100644
Binary files /dev/null and b/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_2.7.10.msgpack differ
diff --git a/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_3.4.3.msgpack b/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_3.4.3.msgpack
new file mode 100644
Binary files /dev/null and b/pandas/io/tests/data/legacy_msgpack/0.16.2/0.16.2_x86_64_linux_3.4.3.msgpack differ
diff --git a/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_2.7.10.pickle b/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_2.7.10.pickle
new file mode 100644
Binary files /dev/null and b/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_2.7.10.pickle differ
diff --git a/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_3.4.3.pickle b/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_3.4.3.pickle
new file mode 100644
Binary files /dev/null and b/pandas/io/tests/data/legacy_pickle/0.16.2/0.16.2_x86_64_linux_3.4.3.pickle differ
diff --git a/pandas/io/tests/generate_legacy_pickles.py b/pandas/io/tests/generate_legacy_pickles.py
deleted file mode 100644
--- a/pandas/io/tests/generate_legacy_pickles.py
+++ /dev/null
@@ -1,167 +0,0 @@
-""" self-contained to write legacy pickle files """
-from __future__ import print_function
-
-
-def _create_sp_series():
-
-    import numpy as np
-    from pandas import SparseSeries
-
-    nan = np.nan
-
-    # nan-based
-    arr = np.arange(15, dtype=np.float64)
-    arr[7:12] = nan
-    arr[-1:] = nan
-
-    bseries = SparseSeries(arr, kind='block')
-    bseries.name = 'bseries'
-    return bseries
-
-def _create_sp_tsseries():
-
-    import numpy as np
-    from pandas import bdate_range, SparseTimeSeries
-
-    nan = np.nan
-
-    # nan-based
-    arr = np.arange(15, dtype=np.float64)
-    arr[7:12] = nan
-    arr[-1:] = nan
-
-    date_index = bdate_range('1/1/2011', periods=len(arr))
-    bseries = SparseTimeSeries(arr, index=date_index, kind='block')
-    bseries.name = 'btsseries'
-    return bseries
-
-def _create_sp_frame():
-    import numpy as np
-    from pandas import bdate_range, SparseDataFrame
-
-    nan = np.nan
-
-    data = {'A': [nan, nan, nan, 0, 1, 2, 3, 4, 5, 6],
-            'B': [0, 1, 2, nan, nan, nan, 3, 4, 5, 6],
-            'C': np.arange(10).astype(np.int64),
-            'D': [0, 1, 2, 3, 4, 5, nan, nan, nan, nan]}
-
-    dates = bdate_range('1/1/2011', periods=10)
-    return SparseDataFrame(data, index=dates)
-
-def create_data():
-    """ create the pickle data """
-
-    from distutils.version import LooseVersion
-    import numpy as np
-    import pandas
-    from pandas import (Series,TimeSeries,DataFrame,Panel,
-                        SparseSeries,SparseTimeSeries,SparseDataFrame,SparsePanel,
-                        Index,MultiIndex,PeriodIndex,
-                        date_range,period_range,bdate_range,Timestamp,Categorical)
-    nan = np.nan
-
-    data = {
-        'A': [0., 1., 2., 3., np.nan],
-        'B': [0, 1, 0, 1, 0],
-        'C': ['foo1', 'foo2', 'foo3', 'foo4', 'foo5'],
-        'D': date_range('1/1/2009', periods=5),
-        'E' : [0., 1, Timestamp('20100101'),'foo',2.],
-        }
-
-    index = dict(int = Index(np.arange(10)),
-                 date = date_range('20130101',periods=10),
-                 period = period_range('2013-01-01', freq='M', periods=10))
-
-    mi = dict(reg2 = MultiIndex.from_tuples(tuple(zip(*[['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux'],
-                                                      ['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two']])),
-                                                 names=['first', 'second']))
-    series = dict(float = Series(data['A']),
-                  int = Series(data['B']),
-                  mixed = Series(data['E']),
-                  ts = TimeSeries(np.arange(10).astype(np.int64),index=date_range('20130101',periods=10)),
-                  mi = Series(np.arange(5).astype(np.float64),index=MultiIndex.from_tuples(tuple(zip(*[[1,1,2,2,2],
-                                                                                                    [3,4,3,4,5]])),
-                                                                                           names=['one','two'])),
-                  dup=Series(np.arange(5).astype(np.float64), index=['A', 'B', 'C', 'D', 'A']),
-                  cat=Series(Categorical(['foo', 'bar', 'baz'])))
-
-    frame = dict(float = DataFrame(dict(A = series['float'], B = series['float'] + 1)),
-                 int = DataFrame(dict(A = series['int']  , B = series['int']   + 1)),
-                 mixed = DataFrame(dict([ (k,data[k]) for k in ['A','B','C','D']])),
-                 mi = DataFrame(dict(A = np.arange(5).astype(np.float64), B = np.arange(5).astype(np.int64)),
-                                index=MultiIndex.from_tuples(tuple(zip(*[['bar','bar','baz','baz','baz'],
-                                                                       ['one','two','one','two','three']])),
-                                                             names=['first','second'])),
-                 dup=DataFrame(np.arange(15).reshape(5, 3).astype(np.float64),
-                               columns=['A', 'B', 'A']),
-                 cat_onecol=DataFrame(dict(A=Categorical(['foo', 'bar']))),
-                 cat_and_float=DataFrame(dict(A=Categorical(['foo', 'bar', 'baz']),
-                                              B=np.arange(3).astype(np.int64))),
-    )
-    panel = dict(float = Panel(dict(ItemA = frame['float'], ItemB = frame['float']+1)),
-                 dup = Panel(np.arange(30).reshape(3, 5, 2).astype(np.float64),
-                             items=['A', 'B', 'A']))
-
-    if LooseVersion(pandas.__version__) >= '0.14.1':
-        # Pre-0.14.1 versions generated non-unpicklable mixed-type frames and
-        # panels if their columns/items were non-unique.
-        mixed_dup_df = DataFrame(data)
-        mixed_dup_df.columns = list("ABCDA")
-
-        mixed_dup_panel = Panel(dict(ItemA=frame['float'], ItemB=frame['int']))
-        mixed_dup_panel.items = ['ItemA', 'ItemA']
-
-        frame['mixed_dup'] = mixed_dup_df
-        panel['mixed_dup'] = mixed_dup_panel
-
-    return dict( series = series,
-                 frame = frame,
-                 panel = panel,
-                 index = index,
-                 mi = mi,
-                 sp_series = dict(float = _create_sp_series(),
-                                  ts = _create_sp_tsseries()),
-                 sp_frame = dict(float = _create_sp_frame())
-                 )
-
-def write_legacy_pickles():
-
-    # force our cwd to be the first searched
-    import sys
-    sys.path.insert(0,'.')
-
-    import os, os.path
-    import numpy as np
-    import pandas
-    import pandas.util.testing as tm
-    import platform as pl
-
-    # make sure we are < 0.13 compat (in py3)
-    try:
-        from pandas.compat import zip, cPickle as pickle
-    except:
-        import pickle
-
-    version = pandas.__version__
-    if len(sys.argv) != 2:
-        exit("Specify output directory: generate_legacy_pickles.py <output_dir>")
-
-    output_dir = str(sys.argv[1])
-
-    print("This script generates a pickle file for the current arch, system, and python version")
-    print("  pandas version: {0}".format(version))
-    print("  output dir    : {0}".format(output_dir))
-
-    # construct a reasonable platform name
-    f = '_'.join([ str(version), str(pl.machine()), str(pl.system().lower()), str(pl.python_version()) ])
-    pth = '{0}.pickle'.format(f)
-
-    fh = open(os.path.join(output_dir,pth),'wb')
-    pickle.dump(create_data(),fh,pickle.HIGHEST_PROTOCOL)
-    fh.close()
-
-    print("created pickle file: %s" % pth)
-
-if __name__ == '__main__':
-    write_legacy_pickles()
diff --git a/pandas/io/tests/generate_legacy_storage_files.py b/pandas/io/tests/generate_legacy_storage_files.py
new file mode 100644
--- /dev/null
+++ b/pandas/io/tests/generate_legacy_storage_files.py
@@ -0,0 +1,205 @@
+""" self-contained to write legacy storage (pickle/msgpack) files """
+from __future__ import print_function
+from distutils.version import LooseVersion
+from pandas import (Series, TimeSeries, DataFrame, Panel,
+                    SparseSeries, SparseTimeSeries, SparseDataFrame, SparsePanel,
+                    Index, MultiIndex, PeriodIndex, bdate_range, to_msgpack,
+                    date_range, period_range, bdate_range, Timestamp, Categorical)
+import os
+import sys
+import numpy as np
+import pandas
+import pandas.util.testing as tm
+import platform as pl
+
+
+def _create_sp_series():
+    nan = np.nan
+
+    # nan-based
+    arr = np.arange(15, dtype=np.float64)
+    arr[7:12] = nan
+    arr[-1:] = nan
+
+    bseries = SparseSeries(arr, kind='block')
+    bseries.name = 'bseries'
+    return bseries
+
+
+def _create_sp_tsseries():
+    nan = np.nan
+
+    # nan-based
+    arr = np.arange(15, dtype=np.float64)
+    arr[7:12] = nan
+    arr[-1:] = nan
+
+    date_index = bdate_range('1/1/2011', periods=len(arr))
+    bseries = SparseTimeSeries(arr, index=date_index, kind='block')
+    bseries.name = 'btsseries'
+    return bseries
+
+
+def _create_sp_frame():
+    nan = np.nan
+
+    data = {'A': [nan, nan, nan, 0, 1, 2, 3, 4, 5, 6],
+            'B': [0, 1, 2, nan, nan, nan, 3, 4, 5, 6],
+            'C': np.arange(10).astype(np.int64),
+            'D': [0, 1, 2, 3, 4, 5, nan, nan, nan, nan]}
+
+    dates = bdate_range('1/1/2011', periods=10)
+    return SparseDataFrame(data, index=dates)
+
+
+def create_data():
+    """ create the pickle/msgpack data """
+
+    data = {
+        'A': [0., 1., 2., 3., np.nan],
+        'B': [0, 1, 0, 1, 0],
+        'C': ['foo1', 'foo2', 'foo3', 'foo4', 'foo5'],
+        'D': date_range('1/1/2009', periods=5),
+        'E': [0., 1, Timestamp('20100101'), 'foo', 2.]
+    }
+
+    index = dict(int=Index(np.arange(10)),
+                 date=date_range('20130101', periods=10),
+                 period=period_range('2013-01-01', freq='M', periods=10))
+
+    mi = dict(reg2=MultiIndex.from_tuples(tuple(zip(*[['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux'],
+                                                      ['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two']])),
+                                          names=['first', 'second']))
+    series = dict(float=Series(data['A']),
+                  int=Series(data['B']),
+                  mixed=Series(data['E']),
+                  ts=TimeSeries(np.arange(10).astype(np.int64), index=date_range('20130101',periods=10)),
+                  mi=Series(np.arange(5).astype(np.float64),
+                            index=MultiIndex.from_tuples(tuple(zip(*[[1, 1, 2, 2, 2], [3, 4, 3, 4, 5]])),
+                                                         names=['one', 'two'])),
+                  dup=Series(np.arange(5).astype(np.float64), index=['A', 'B', 'C', 'D', 'A']),
+                  cat=Series(Categorical(['foo', 'bar', 'baz'])))
+
+    mixed_dup_df = DataFrame(data)
+    mixed_dup_df.columns = list("ABCDA")
+    frame = dict(float=DataFrame(dict(A=series['float'], B=series['float'] + 1)),
+                 int=DataFrame(dict(A=series['int'], B=series['int'] + 1)),
+                 mixed=DataFrame(dict([(k, data[k]) for k in ['A', 'B', 'C', 'D']])),
+                 mi=DataFrame(dict(A=np.arange(5).astype(np.float64), B=np.arange(5).astype(np.int64)),
+                              index=MultiIndex.from_tuples(tuple(zip(*[['bar', 'bar', 'baz', 'baz', 'baz'],
+                                                                       ['one', 'two', 'one', 'two', 'three']])),
+                                                           names=['first', 'second'])),
+                 dup=DataFrame(np.arange(15).reshape(5, 3).astype(np.float64),
+                               columns=['A', 'B', 'A']),
+                 cat_onecol=DataFrame(dict(A=Categorical(['foo', 'bar']))),
+                 cat_and_float=DataFrame(dict(A=Categorical(['foo', 'bar', 'baz']),
+                                              B=np.arange(3).astype(np.int64))),
+                 mixed_dup=mixed_dup_df)
+
+    mixed_dup_panel = Panel(dict(ItemA=frame['float'], ItemB=frame['int']))
+    mixed_dup_panel.items = ['ItemA', 'ItemA']
+    panel = dict(float=Panel(dict(ItemA=frame['float'], ItemB=frame['float'] + 1)),
+                 dup=Panel(np.arange(30).reshape(3, 5, 2).astype(np.float64),
+                           items=['A', 'B', 'A']),
+                 mixed_dup=mixed_dup_panel)
+
+    return dict(series=series,
+                frame=frame,
+                panel=panel,
+                index=index,
+                mi=mi,
+                sp_series=dict(float=_create_sp_series(),
+                               ts=_create_sp_tsseries()),
+                sp_frame=dict(float=_create_sp_frame()))
+
+
+def create_pickle_data():
+    data = create_data()
+
+    # Pre-0.14.1 versions generated non-unpicklable mixed-type frames and
+    # panels if their columns/items were non-unique.
+    if LooseVersion(pandas.__version__) < '0.14.1':
+        del data['frame']['mixed_dup']
+        del data['panel']['mixed_dup']
+    return data
+
+
+def create_msgpack_data():
+    data = create_data()
+    if LooseVersion(pandas.__version__) < '0.17.0':
+        del data['frame']['mixed_dup']
+        del data['panel']['mixed_dup']
+        del data['frame']['dup']
+        del data['panel']['dup']
+    # Not supported
+    del data['sp_series']
+    del data['sp_frame']
+    del data['series']['cat']
+    del data['frame']['cat_onecol']
+    del data['frame']['cat_and_float']
+    return data
+
+
+def platform_name():
+    return '_'.join([str(pandas.__version__), str(pl.machine()), str(pl.system().lower()), str(pl.python_version())])
+
+
+def write_legacy_pickles(output_dir):
+
+    # make sure we are < 0.13 compat (in py3)
+    try:
+        from pandas.compat import zip, cPickle as pickle
+    except:
+        import pickle
+
+    version = pandas.__version__
+
+    print("This script generates a storage file for the current arch, system, and python version")
+    print("  pandas version: {0}".format(version))
+    print("  output dir    : {0}".format(output_dir))
+    print("  storage format: pickle")
+
+    pth = '{0}.pickle'.format(platform_name())
+
+    fh = open(os.path.join(output_dir, pth), 'wb')
+    pickle.dump(create_pickle_data(), fh, pickle.HIGHEST_PROTOCOL)
+    fh.close()
+
+    print("created pickle file: %s" % pth)
+
+
+def write_legacy_msgpack(output_dir):
+
+    version = pandas.__version__
+
+    print("This script generates a storage file for the current arch, system, and python version")
+    print("  pandas version: {0}".format(version))
+    print("  output dir    : {0}".format(output_dir))
+    print("  storage format: msgpack")
+
+    pth = '{0}.msgpack'.format(platform_name())
+    to_msgpack(os.path.join(output_dir, pth), create_msgpack_data())
+
+    print("created msgpack file: %s" % pth)
+
+
+def write_legacy_file():
+    # force our cwd to be the first searched
+    sys.path.insert(0, '.')
+
+    if len(sys.argv) != 3:
+        exit("Specify output directory and storage type: generate_legacy_storage_files.py <output_dir> <storage_type>")
+
+    output_dir = str(sys.argv[1])
+    storage_type = str(sys.argv[2])
+
+    if storage_type == 'pickle':
+        write_legacy_pickles(output_dir=output_dir)
+    elif storage_type == 'msgpack':
+        write_legacy_msgpack(output_dir=output_dir)
+    else:
+        exit("storage_type must be one of {'pickle', 'msgpack'}")
+
+
+if __name__ == '__main__':
+    write_legacy_file()
diff --git a/pandas/io/tests/test_packers.py b/pandas/io/tests/test_packers.py
--- a/pandas/io/tests/test_packers.py
+++ b/pandas/io/tests/test_packers.py
@@ -1,5 +1,6 @@
 import nose
 
+import os
 import datetime
 import numpy as np
 import sys
@@ -11,7 +12,7 @@
                     date_range, period_range, Index, SparseSeries, SparseDataFrame,
                     SparsePanel)
 import pandas.util.testing as tm
-from pandas.util.testing import ensure_clean
+from pandas.util.testing import ensure_clean, assert_index_equal
 from pandas.tests.test_series import assert_series_equal
 from pandas.tests.test_frame import assert_frame_equal
 from pandas.tests.test_panel import assert_panel_equal
@@ -39,6 +40,8 @@ def check_arbitrary(a, b):
         assert_frame_equal(a, b)
     elif isinstance(a, Series):
         assert_series_equal(a, b)
+    elif isinstance(a, Index):
+        assert_index_equal(a, b)
     else:
         assert(a == b)
 
@@ -396,6 +399,24 @@ def tests_datetimeindex_freq_issue(self):
         result = self.encode_decode(df)
         assert_frame_equal(result, df)
 
+    def test_dataframe_duplicate_column_names(self):
+
+        # GH 9618
+        expected_1 = DataFrame(columns=['a', 'a'])
+        expected_2 = DataFrame(columns=[1]*100)
+        expected_2.loc[0] = np.random.randn(100)
+        expected_3 = DataFrame(columns=[1, 1])
+        expected_3.loc[0] = ['abc', np.nan]
+
+        result_1 = self.encode_decode(expected_1)
+        result_2 = self.encode_decode(expected_2)
+        result_3 = self.encode_decode(expected_3)
+
+        assert_frame_equal(result_1, expected_1)
+        assert_frame_equal(result_2, expected_2)
+        assert_frame_equal(result_3, expected_3)
+
+
 class TestSparse(TestPackers):
 
     def _check_roundtrip(self, obj, comparator, **kwargs):
@@ -496,6 +517,58 @@ def test_compression_blosc(self):
             assert_frame_equal(self.frame[k], i_rec[k])
 
 
+class TestMsgpack():
+    """
+    How to add msgpack tests:
+
+    1. Install pandas version intended to output the msgpack.
+
+    2. Execute "generate_legacy_storage_files.py" to create the msgpack.
+    $ python generate_legacy_storage_files.py <output_dir> msgpack
+
+    3. Move the created pickle to "data/legacy_msgpack/<version>" directory.
+
+    NOTE: TestMsgpack can't be a subclass of tm.Testcase to use test generator.
+    http://stackoverflow.com/questions/6689537/nose-test-generators-inside-class
+    """
+    def setUp(self):
+        from pandas.io.tests.generate_legacy_storage_files import create_msgpack_data
+        self.data = create_msgpack_data()
+        self.path = u('__%s__.msgpack' % tm.rands(10))
+
+    def compare(self, vf):
+        data = read_msgpack(vf)
+        for typ, dv in data.items():
+            for dt, result in dv.items():
+                try:
+                    expected = self.data[typ][dt]
+                except KeyError:
+                    continue
+                check_arbitrary(result, expected)
+
+        return data
+
+    def read_msgpacks(self, version):
+
+        pth = tm.get_data_path('legacy_msgpack/{0}'.format(str(version)))
+        n = 0
+        for f in os.listdir(pth):
+            vf = os.path.join(pth, f)
+            self.compare(vf)
+            n += 1
+        assert n > 0, 'Msgpack files are not tested'
+
+    def test_msgpack(self):
+        msgpack_path = tm.get_data_path('legacy_msgpack')
+        n = 0
+        for v in os.listdir(msgpack_path):
+            pth = os.path.join(msgpack_path, v)
+            if os.path.isdir(pth):
+                yield self.read_msgpacks, v
+            n += 1
+        assert n > 0, 'Msgpack files are not tested'
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
diff --git a/pandas/io/tests/test_pickle.py b/pandas/io/tests/test_pickle.py
--- a/pandas/io/tests/test_pickle.py
+++ b/pandas/io/tests/test_pickle.py
@@ -24,8 +24,8 @@ class TestPickle():
 
     1. Install pandas version intended to output the pickle.
 
-    2. Execute "generate_legacy_pkcles.py" to create the pickle.
-    $ python generate_legacy_pickles.py <version> <output_dir>
+    2. Execute "generate_legacy_storage_files.py" to create the pickle.
+    $ python generate_legacy_storage_files.py <output_dir> pickle
 
     3. Move the created pickle to "data/legacy_pickle/<version>" directory.
 
@@ -35,8 +35,8 @@ class TestPickle():
     _multiprocess_can_split_ = True
 
     def setUp(self):
-        from pandas.io.tests.generate_legacy_pickles import create_data
-        self.data = create_data()
+        from pandas.io.tests.generate_legacy_storage_files import create_pickle_data
+        self.data = create_pickle_data()
         self.path = u('__%s__.pickle' % tm.rands(10))
 
     def compare_element(self, typ, result, expected):