Repository: pandas-dev/pandas

Issue:
Cannot serialize objects with IntervalIndex using msgpack
#### Code Sample, a copy-pastable example if possible
From 
```python
import numpy as np
import pandas as pd
ser = pd.Series(np.arange(10), pd.interval_range(0, periods=10))
ser.to_msgpack('C:/temp/blah.mpk')

-->
TypeError                                 Traceback (most recent call last)
<ipython-input-8-0da7c2fd5b28> in <module>()
----> 1 pd.Series(np.arange(10), pd.interval_range(0, periods=10)).to_msgpack('C:/temp/blah.mpk')

C:\projects\pandas-dk\pandas\core\generic.py in to_msgpack(self, path_or_buf, encoding, **kwargs)
   1860         from pandas.io import packers
   1861         return packers.to_msgpack(path_or_buf, self, encoding=encoding,
-> 1862                                   **kwargs)
   1863
   1864     def to_sql(self, name, con, schema=None, if_exists='fail', index=True,

C:\projects\pandas-dk\pandas\io\packers.py in to_msgpack(path_or_buf, *args, **kwargs)
    152     if isinstance(path_or_buf, compat.string_types):
    153         with open(path_or_buf, mode) as fh:
--> 154             writer(fh)
    155     elif path_or_buf is None:
    156         buf = compat.BytesIO()

C:\projects\pandas-dk\pandas\io\packers.py in writer(fh)
    147     def writer(fh):
    148         for a in args:
--> 149             fh.write(pack(a, **kwargs))
    150
    151     path_or_buf = _stringify_path(path_or_buf)

C:\projects\pandas-dk\pandas\io\packers.py in pack(o, default, encoding, unicode_errors, use_single_float, autoreset, use_bin_type)
    700                   use_single_float=use_single_float,
    701                   autoreset=autoreset,
--> 702                   use_bin_type=use_bin_type).pack(o)
    703
    704

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer.pack()
    229             return ret
    230
--> 231     cpdef pack(self, object obj):
    232         cdef int ret
    233         ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer.pack()
    231     cpdef pack(self, object obj):
    232         cdef int ret
--> 233         ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
    234         if ret == -1:
    235             raise MemoryError

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer._pack()
    190                         ret = self._pack(k, nest_limit - 1)
    191                         if ret != 0: break
--> 192                         ret = self._pack(v, nest_limit - 1)
    193                         if ret != 0: break
    194             elif PyDict_Check(o):

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer._pack()
    190                         ret = self._pack(k, nest_limit - 1)
    191                         if ret != 0: break
--> 192                         ret = self._pack(v, nest_limit - 1)
    193                         if ret != 0: break
    194             elif PyDict_Check(o):

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer._pack()
    219                 if ret == 0:
    220                     for v in o:
--> 221                         ret = self._pack(v, nest_limit - 1)
    222                         if ret != 0: break
    223             elif not default_used and self._default:

C:\projects\pandas-dk\pandas\io\msgpack\_packer.pyx in pandas.io.msgpack._packer.Packer._pack()
    226                 continue
    227             else:
--> 228                 raise TypeError("can't serialize {thing!r}".format(thing=o))
    229             return ret
    230

TypeError: can't serialize Interval(0, 1, closed='right')
```
#### Problem description
Msgpack cannot serialize currently objects which have IntervalIndex. I am not sure if this is related in any way to #13463; there the issue was just serialization of a "Period", but PeriodIndex works. Here in the context of trying to serialize IntervalIndex it appears it's actually trying to serialize the Interval object.

#### Expected Output
Successful serialization without error.

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: e3b87c1843ef1a2375faff737701a13ee96f4879
python: 3.6.4.final.0
python-bits: 64
OS: Windows
OS-release: 7
machine: AMD64
processor: Intel64 Family 6 Model 62 Stepping 4, GenuineIntel
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.23.0.dev0+422.ge3b87c1
pytest: 3.3.2
pip: 9.0.1
setuptools: 38.4.0
Cython: 0.27.3
numpy: 1.14.0
scipy: 1.0.0
pyarrow: 0.8.0
xarray: 0.10.0
IPython: 6.2.1
sphinx: 1.6.6
patsy: 0.5.0
dateutil: 2.6.1
pytz: 2017.3
blosc: None
bottleneck: 1.2.1
tables: 3.4.2
numexpr: 2.6.4
feather: 0.4.0
matplotlib: 2.1.2
openpyxl: 2.4.10
xlrd: 1.1.0
xlwt: 1.3.0
xlsxwriter: 1.0.2
lxml: 4.1.1
bs4: 4.6.0
html5lib: 1.0.1
sqlalchemy: 1.2.1
pymysql: 0.7.11.None
psycopg2: None
jinja2: 2.10
s3fs: 0.1.2
fastparquet: 0.1.4
pandas_gbq: None
pandas_datareader: None

</details>

to_msgpack() fails when data has "period"
to_msgpack() fails when the data has "period".  This works in 0.16.2
#### ISSUE

```
import pandas as pd
pd.DataFrame({'C': pd.period_range('2015-01-01', periods=3)}).to_msgpack()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-65-8e5ad9cd6036> in <module>()
----> 1 pd.DataFrame({'C': pd.period_range('2015-01-01', periods=3)}).to_msgpack()

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/core/generic.pyc in to_msgpack(self, path_or_buf, encoding, **kwargs)
   1120         from pandas.io import packers
   1121         return packers.to_msgpack(path_or_buf, self, encoding=encoding,
-> 1122                                   **kwargs)
   1123
   1124     def to_sql(self, name, con, flavor='sqlite', schema=None, if_exists='fail',

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/io/packers.pyc in to_msgpack(path_or_buf, *args, **kwargs)
    152     elif path_or_buf is None:
    153         buf = compat.BytesIO()
--> 154         writer(buf)
    155         return buf.getvalue()
    156     else:

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/io/packers.pyc in writer(fh)
    145     def writer(fh):
    146         for a in args:
--> 147             fh.write(pack(a, **kwargs))
    148
    149     if isinstance(path_or_buf, compat.string_types):

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/io/packers.pyc in pack(o, default, encoding, unicode_errors, use_single_float, autoreset, use_bin_type)
    699                   use_single_float=use_single_float,
    700                   autoreset=autoreset,
--> 701                   use_bin_type=use_bin_type).pack(o)
    702
    703

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer.pack (pandas/msgpack/_packer.cpp:3434)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer.pack (pandas/msgpack/_packer.cpp:3276)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer._pack (pandas/msgpack/_packer.cpp:2433)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer._pack (pandas/msgpack/_packer.cpp:3006)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer._pack (pandas/msgpack/_packer.cpp:2433)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer._pack (pandas/msgpack/_packer.cpp:3006)()

pandas/msgpack/_packer.pyx in pandas.msgpack._packer.Packer._pack (pandas/msgpack/_packer.cpp:3088)()

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/io/packers.pyc in encode(obj)
    510         return {u'typ': u'period',
    511                 u'ordinal': obj.ordinal,
--> 512                 u'freq': u(obj.freq)}
    513     elif isinstance(obj, BlockIndex):
    514         return {u'typ': u'block_index',

/auto/energymdl2/anaconda/envs/commod_20160516_pd18/lib/python2.7/site-packages/pandas/compat/__init__.pyc in u(s)
    262
    263     def u(s):
--> 264         return unicode(s, "unicode_escape")
    265
    266     def u_safe(s):

TypeError: coercing to Unicode: need string or buffer, Day found

```
#### Expected Output

Works on 0.16.2

```
In [30]: pd.DataFrame({'C': pd.period_range('2015-01-01', periods=3)}).to_msgpack()
Out[30]: '\x84\xa6blocks\x91\x86\xa5items\x86\xa4name\xc0\xa5dtype\x11\xa8compress\xc0\xa4data\x91\xa1C\xa5klass\xa5Index\xa3typ\xa5index\xa8compress\xc0\xa5shape\x92\x01\x03\xa6values\x93\x83\xa7ordinal\xcd@4\xa4freq\xa1D\xa3typ\xa6period\x83\xa7ordinal\xcd@5\xa4freq\xa1D\xa3typ\xa6period\x83\xa7ordinal\xcd@6\xa4freq\xa1D\xa3typ\xa6period\xa5klass\xabObjectBlock\xa5dtype\x11\xa4axes\x92\x86\xa4name\xc0\xa5dtype\x11\xa8compress\xc0\xa4data\x91\xa1C\xa5klass\xa5Index\xa3typ\xa5index\x86\xa4name\xc0\xa5dtype\x07\xa8compress\xc0\xa4data\xb8\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\xa5klass\xaaInt64Index\xa3typ\xa5index\xa3typ\xadblock_manager\xa5klass\xa9DataFrame'
```
#### INSTALLED VERSIONS

```

------------------
commit: None
python: 2.7.11.final.0
python-bits: 64
OS: Linux
OS-release: 2.6.32-573.7.1.el6.x86_64
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: C
LANG: en_US.UTF-8

pandas: 0.18.1
nose: 1.3.7
pip: 8.1.1
setuptools: 20.7.0
Cython: 0.24
numpy: 1.10.4
scipy: 0.17.0
statsmodels: 0.6.1
xarray: 0.7.2
IPython: 4.1.2
sphinx: 1.3.5
patsy: 0.4.1
dateutil: 2.5.2
pytz: 2016.4
blosc: None
bottleneck: 1.0.0
tables: 3.2.2
numexpr: 2.5.2
matplotlib: 1.4.3
openpyxl: 2.3.2
xlrd: 0.9.4
xlwt: 1.0.0
xlsxwriter: 0.8.4
lxml: 3.6.0
bs4: 4.3.2
html5lib: 0.999
httplib2: 0.9.2
apiclient: 1.5.0
sqlalchemy: 1.0.12
pymysql: None
psycopg2: None
jinja2: 2.8
boto: 2.39.0
pandas_datareader: None
```



Discussion:
Yes, I think some additional logic needs to be added around these two spots to encode & decode IntervalIndex - can look at treatment of others indexes for a template
https://github.com/pandas-dev/pandas/blob/a7a7f8c1101aed1a9d37abbbcd80f77da414f0a8/pandas/io/packers.py#L374

https://github.com/pandas-dev/pandas/blob/a7a7f8c1101aed1a9d37abbbcd80f77da414f0a8/pandas/io/packers.py#L560

PR welcome if you're interested
cc @jschendel 

So periods work as part of a Period index, but general support for `object` that is non-string is not very well covered in tests. Currently `Period` is an `object` dtype.

quickest way to have this fixed is to do a patch and submit as a PR.

For Period, we would patch with obj.freqstr, instead of obj.freq.

```
    509     elif isinstance(obj, Period):
    510         return {u'typ': u'period',
    511                 u'ordinal': obj.ordinal,
1-> 512                 u'freq': u(obj.freq)}
```

yes this is prob a simple fix. But needs some adequate testing (as well as prob a checkin of some generated msgpacks on older versions, use `pandas/io/tests/generate_legacy_storage_files.py` to generate from a working version)


Resolving patch:
diff --git a/pandas/io/packers.py b/pandas/io/packers.py
--- a/pandas/io/packers.py
+++ b/pandas/io/packers.py
@@ -55,7 +55,8 @@
 from pandas import (Timestamp, Period, Series, DataFrame,  # noqa
                     Index, MultiIndex, Float64Index, Int64Index,
                     Panel, RangeIndex, PeriodIndex, DatetimeIndex, NaT,
-                    Categorical, CategoricalIndex)
+                    Categorical, CategoricalIndex, IntervalIndex, Interval,
+                    TimedeltaIndex)
 from pandas.core.sparse.api import SparseSeries, SparseDataFrame
 from pandas.core.sparse.array import BlockIndex, IntIndex
 from pandas.core.generic import NDFrame
@@ -401,6 +402,13 @@ def encode(obj):
                     u'freq': u_safe(getattr(obj, 'freqstr', None)),
                     u'tz': tz,
                     u'compress': compressor}
+        elif isinstance(obj, IntervalIndex):
+            return {u'typ': u'interval_index',
+                    u'klass': u(obj.__class__.__name__),
+                    u'name': getattr(obj, 'name', None),
+                    u'left': getattr(obj, '_left', None),
+                    u'right': getattr(obj, '_right', None),
+                    u'closed': getattr(obj, '_closed', None)}
         elif isinstance(obj, MultiIndex):
             return {u'typ': u'multi_index',
                     u'klass': u(obj.__class__.__name__),
@@ -513,7 +521,12 @@ def encode(obj):
     elif isinstance(obj, Period):
         return {u'typ': u'period',
                 u'ordinal': obj.ordinal,
-                u'freq': u(obj.freq)}
+                u'freq': u_safe(obj.freqstr)}
+    elif isinstance(obj, Interval):
+        return {u'typ': u'interval',
+                u'left': obj.left,
+                u'right': obj.right,
+                u'closed': obj.closed}
     elif isinstance(obj, BlockIndex):
         return {u'typ': u'block_index',
                 u'klass': u(obj.__class__.__name__),
@@ -597,12 +610,19 @@ def decode(obj):
             result = result.tz_localize('UTC').tz_convert(tz)
         return result
 
+    elif typ == u'interval_index':
+        return globals()[obj[u'klass']].from_arrays(obj[u'left'],
+                                                    obj[u'right'],
+                                                    obj[u'closed'],
+                                                    name=obj[u'name'])
     elif typ == u'category':
         from_codes = globals()[obj[u'klass']].from_codes
         return from_codes(codes=obj[u'codes'],
                           categories=obj[u'categories'],
                           ordered=obj[u'ordered'])
 
+    elif typ == u'interval':
+        return Interval(obj[u'left'], obj[u'right'], obj[u'closed'])
     elif typ == u'series':
         dtype = dtype_for(obj[u'dtype'])
         pd_dtype = pandas_dtype(dtype)


Test patch:
diff --git a/pandas/tests/io/generate_legacy_storage_files.py b/pandas/tests/io/generate_legacy_storage_files.py
--- a/pandas/tests/io/generate_legacy_storage_files.py
+++ b/pandas/tests/io/generate_legacy_storage_files.py
@@ -287,7 +287,8 @@ def create_msgpack_data():
     del data['frame']['cat_onecol']
     del data['frame']['cat_and_float']
     del data['scalars']['period']
-    del data['index']['interval']
+    if _loose_version < LooseVersion('0.23.0'):
+        del data['index']['interval']
     del data['offsets']
     return _u(data)
 
diff --git a/pandas/tests/io/test_packers.py b/pandas/tests/io/test_packers.py
--- a/pandas/tests/io/test_packers.py
+++ b/pandas/tests/io/test_packers.py
@@ -10,7 +10,8 @@
 from pandas import compat
 from pandas.compat import u, PY3
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
-                    date_range, period_range, Index, Categorical)
+                    date_range, period_range, Index, Categorical,
+                    Period, Interval)
 from pandas.errors import PerformanceWarning
 from pandas.io.packers import to_msgpack, read_msgpack
 import pandas.util.testing as tm
@@ -317,6 +318,19 @@ def test_timedeltas(self):
             i_rec = self.encode_decode(i)
             assert i == i_rec
 
+    def test_periods(self):
+        # 13463
+        for i in [Period('2010-09', 'M'), Period('2014-Q1', 'Q')]:
+            i_rec = self.encode_decode(i)
+            assert i == i_rec
+
+    def test_intervals(self):
+        # 19967
+        for i in [Interval(0, 1), Interval(0, 1, 'left'),
+                  Interval(10, 25., 'right')]:
+            i_rec = self.encode_decode(i)
+            assert i == i_rec
+
 
 class TestIndex(TestPackers):
 
@@ -334,7 +348,9 @@ def setup_method(self, method):
             'period': Index(period_range('2012-1-1', freq='M', periods=3)),
             'date2': Index(date_range('2013-01-1', periods=10)),
             'bdate': Index(bdate_range('2013-01-02', periods=10)),
-            'cat': tm.makeCategoricalIndex(100)
+            'cat': tm.makeCategoricalIndex(100),
+            'interval': tm.makeIntervalIndex(100),
+            'timedelta': tm.makeTimedeltaIndex(100, 'H')
         }
 
         self.mi = {