Repository: pandas-dev/pandas

Issue:
read_pickle much slower in v0.13 (not using cPickle when compat=False)
See discussion here http://stackoverflow.com/questions/23122180/is-pandas-read-pickle-performance-crippled-in-version-0-13

My test dataset has pickled file size 1.6Gb, and contains about 13 million records.

With 0.12 the file takes 146s to load; with 0.13 is takes 982s (about 6.7x longer).

Using cProfile, you can see that v0.13 always uses native python pickle to load, even when compat=False.  In 0.12 it uses cPickle to load.   Seems like something is wrong with the logic in pandas/compat/pickle_compat.py

A workaround (if you know you don't need compatibility mode) is to use cPickle.load(open('foo.pickle')) instead of pandas.read_pickle('foo.pickle').



Discussion:
Thanks for the report! I'll try and take a look. There were some weird
unicode issues to handle but we definitely don't want to sacrifice pickle
performance


Resolving patch:
diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py
--- a/pandas/io/pickle.py
+++ b/pandas/io/pickle.py
@@ -34,16 +34,28 @@ def read_pickle(path):
     """
 
     def try_read(path, encoding=None):
+        # try with cPickle
         # try with current pickle, if we have a Type Error then
         # try with the compat pickle to handle subclass changes
         # pass encoding only if its not None as py2 doesn't handle
         # the param
+
+        # cpickle
+        # GH 6899
         try:
             with open(path, 'rb') as fh:
-                return pc.load(fh, encoding=encoding, compat=False)
+                return pkl.load(fh)
         except:
-            with open(path, 'rb') as fh:
-                return pc.load(fh, encoding=encoding, compat=True)
+
+            # reg/patched pickle
+            try:
+                with open(path, 'rb') as fh:
+                    return pc.load(fh, encoding=encoding, compat=False)
+
+            # compat pickle
+            except:
+                with open(path, 'rb') as fh:
+                    return pc.load(fh, encoding=encoding, compat=True)
 
     try:
         return try_read(path)
diff --git a/vb_suite/packers.py b/vb_suite/packers.py
--- a/vb_suite/packers.py
+++ b/vb_suite/packers.py
@@ -7,6 +7,7 @@
 import os
 import pandas as pd
 from pandas.core import common as com
+from random import randrange
 
 f = '__test__.msg'
 def remove(f):
@@ -15,10 +16,18 @@ def remove(f):
    except:
        pass
 
-index = date_range('20000101',periods=50000,freq='H')
-df = DataFrame({'float1' : randn(50000),
-                'float2' : randn(50000)},
+N=100000
+C=5
+index = date_range('20000101',periods=N,freq='H')
+df = DataFrame(dict([ ("float{0}".format(i),randn(N)) for i in range(C) ]),
                index=index)
+
+N=100000
+C=5
+index = date_range('20000101',periods=N,freq='H')
+df2 = DataFrame(dict([ ("float{0}".format(i),randn(N)) for i in range(C) ]),
+                index=index)
+df2['object'] = ['%08x'%randrange(16**8) for _ in range(N)]
 remove(f)
 """
 
@@ -26,7 +35,7 @@ def remove(f):
 # msgpack
 
 setup = common_setup + """
-df.to_msgpack(f)
+df2.to_msgpack(f)
 """
 
 packers_read_pack = Benchmark("pd.read_msgpack(f)", setup, start_date=start_date)
@@ -34,13 +43,13 @@ def remove(f):
 setup = common_setup + """
 """
 
-packers_write_pack = Benchmark("df.to_msgpack(f)", setup, cleanup="remove(f)", start_date=start_date)
+packers_write_pack = Benchmark("df2.to_msgpack(f)", setup, cleanup="remove(f)", start_date=start_date)
 
 #----------------------------------------------------------------------
 # pickle
 
 setup = common_setup + """
-df.to_pickle(f)
+df2.to_pickle(f)
 """
 
 packers_read_pickle = Benchmark("pd.read_pickle(f)", setup, start_date=start_date)
@@ -48,7 +57,7 @@ def remove(f):
 setup = common_setup + """
 """
 
-packers_write_pickle = Benchmark("df.to_pickle(f)", setup, cleanup="remove(f)", start_date=start_date)
+packers_write_pickle = Benchmark("df2.to_pickle(f)", setup, cleanup="remove(f)", start_date=start_date)
 
 #----------------------------------------------------------------------
 # csv
@@ -68,7 +77,7 @@ def remove(f):
 # hdf store
 
 setup = common_setup + """
-df.to_hdf(f,'df')
+df2.to_hdf(f,'df')
 """
 
 packers_read_hdf_store = Benchmark("pd.read_hdf(f,'df')", setup, start_date=start_date)
@@ -76,13 +85,13 @@ def remove(f):
 setup = common_setup + """
 """
 
-packers_write_hdf_store = Benchmark("df.to_hdf(f,'df')", setup, cleanup="remove(f)", start_date=start_date)
+packers_write_hdf_store = Benchmark("df2.to_hdf(f,'df')", setup, cleanup="remove(f)", start_date=start_date)
 
 #----------------------------------------------------------------------
 # hdf table
 
 setup = common_setup + """
-df.to_hdf(f,'df',table=True)
+df2.to_hdf(f,'df',table=True)
 """
 
 packers_read_hdf_table = Benchmark("pd.read_hdf(f,'df')", setup, start_date=start_date)
@@ -90,7 +99,7 @@ def remove(f):
 setup = common_setup + """
 """
 
-packers_write_hdf_table = Benchmark("df.to_hdf(f,'df',table=True)", setup, cleanup="remove(f)", start_date=start_date)
+packers_write_hdf_table = Benchmark("df2.to_hdf(f,'df',table=True)", setup, cleanup="remove(f)", start_date=start_date)
 
 #----------------------------------------------------------------------
 # json


Test patch: