Repository: pandas-dev/pandas

Issue:
Msgpack - ValueError: buffer source array is read-only 
I get the Value error when processing data using pandas. I followed the following steps:
1.  convert to msgpack format with compress flag
2. subsequently read file into a dataframe
3. push to sql table with to_sql

On the third step i get ValueError: buffer source array is read-only.

This problem does not arise if I wrap the read_msgpack call inside a pandas.concat

Example

``` python

import pandas as pd
import numpy as np

from sqlalchemy import create_engine

eng = create_engine("sqlite:///:memory:")

df1 = pd.DataFrame({ 'A' : 1.,
                                   'B' : pd.Timestamp('20130102'),
                                   'C' : pd.Series(1,index=list(range(4)),dtype='float32'),
                                   'D' : np.array([3] * 4,dtype='int32'),
                                   'E' : 'foo' })

df1.to_msgpack('test.msgpack', compress='zlib')
df2 = pd.read_msgpack('test.msgpack')

df2.to_sql('test', eng, if_exists='append', chunksize=1000) # throws value error

df2 = pd.cooncat([pd.read_msgpack('test.msgpack')])

df2.to_sql('test', eng, if_exists='append', chunksize=1000) # works
```

This happens with both blosc and zlib compression. While I have found a solution, this behaviour seems very odd and for very large files there is a small performance hit.

edit: @TomAugspurger changed the sql engine to sqlite



Discussion:
pls pd.show_versions()

replace `eng = create_engine("mysql+mysqldb://user:pass@localhost/dbname")` with `eng = create_engine("sqlite:///:memory:")` to make this easier to reproduce (still raises)

Output of pd.show_versions()

## INSTALLED VERSIONS

commit: None
python: 3.5.0.final.0
python-bits: 64
OS: Linux
OS-release: 4.2.0-21-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_IN

pandas: 0.17.1
nose: 1.3.7
pip: 7.1.2
setuptools: 18.2
Cython: 0.23.4
numpy: 1.10.2
scipy: 0.16.1
statsmodels: None
IPython: 4.0.1
sphinx: None
patsy: None
dateutil: 2.4.2
pytz: 2015.7
blosc: 1.2.8
bottleneck: None
tables: None
numexpr: None
matplotlib: 1.5.0
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: 4.4.1
html5lib: None
httplib2: None
apiclient: None
sqlalchemy: 1.0.10
pymysql: None
psycopg2: None
Jinja2: None

I think we need to tell numpy to take ownership of the data, maybe `np.array` with `copy=False` around the `np.frombuffer`. @shoyer how does one normally do this?

```
In [9]: df2._data.blocks[0].values.flags
Out[9]: 
  C_CONTIGUOUS : True
  F_CONTIGUOUS : True
  OWNDATA : False
  WRITEABLE : False
  ALIGNED : False
  UPDATEIFCOPY : False

In [10]: df1._data.blocks[0].values.flags
Out[10]: 
  C_CONTIGUOUS : True
  F_CONTIGUOUS : True
  OWNDATA : True
  WRITEABLE : True
  ALIGNED : True
  UPDATEIFCOPY : False
```


Resolving patch:
diff --git a/pandas/core/window.py b/pandas/core/window.py
--- a/pandas/core/window.py
+++ b/pandas/core/window.py
@@ -965,6 +965,7 @@ def corr(self, other=None, pairwise=None, **kwargs):
    Use a standard estimation bias correction
 """
 
+
 class EWM(_Rolling):
     r"""
     Provides exponential weighted functions


Test patch:
diff --git a/pandas/io/tests/test_packers.py b/pandas/io/tests/test_packers.py
--- a/pandas/io/tests/test_packers.py
+++ b/pandas/io/tests/test_packers.py
@@ -9,8 +9,8 @@
 from pandas import compat
 from pandas.compat import u
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
-                    date_range, period_range, Index, SparseSeries, SparseDataFrame,
-                    SparsePanel)
+                    date_range, period_range, Index)
+from pandas.io.packers import to_msgpack, read_msgpack
 import pandas.util.testing as tm
 from pandas.util.testing import (ensure_clean, assert_index_equal,
                                  assert_series_equal,
@@ -23,7 +23,19 @@
 
 nan = np.nan
 
-from pandas.io.packers import to_msgpack, read_msgpack
+try:
+    import blosc  # NOQA
+except ImportError:
+    _BLOSC_INSTALLED = False
+else:
+    _BLOSC_INSTALLED = True
+
+try:
+    import zlib  # NOQA
+except ImportError:
+    _ZLIB_INSTALLED = False
+else:
+    _ZLIB_INSTALLED = True
 
 _multiprocess_can_split_ = False
 
@@ -483,6 +495,14 @@ class TestCompression(TestPackers):
     """
 
     def setUp(self):
+        try:
+            from sqlalchemy import create_engine
+            self._create_sql_engine = create_engine
+        except ImportError:
+            self._SQLALCHEMY_INSTALLED = False
+        else:
+            self._SQLALCHEMY_INSTALLED = True
+
         super(TestCompression, self).setUp()
         data = {
             'A': np.arange(1000, dtype=np.float64),
@@ -508,14 +528,56 @@ def test_compression_zlib(self):
             assert_frame_equal(self.frame[k], i_rec[k])
 
     def test_compression_blosc(self):
-        try:
-            import blosc
-        except ImportError:
+        if not _BLOSC_INSTALLED:
             raise nose.SkipTest('no blosc')
         i_rec = self.encode_decode(self.frame, compress='blosc')
         for k in self.frame.keys():
             assert_frame_equal(self.frame[k], i_rec[k])
 
+    def test_readonly_axis_blosc(self):
+        # GH11880
+        if not _BLOSC_INSTALLED:
+            raise nose.SkipTest('no blosc')
+        df1 = DataFrame({'A': list('abcd')})
+        df2 = DataFrame(df1, index=[1., 2., 3., 4.])
+        self.assertTrue(1 in self.encode_decode(df1['A'], compress='blosc'))
+        self.assertTrue(1. in self.encode_decode(df2['A'], compress='blosc'))
+
+    def test_readonly_axis_zlib(self):
+        # GH11880
+        df1 = DataFrame({'A': list('abcd')})
+        df2 = DataFrame(df1, index=[1., 2., 3., 4.])
+        self.assertTrue(1 in self.encode_decode(df1['A'], compress='zlib'))
+        self.assertTrue(1. in self.encode_decode(df2['A'], compress='zlib'))
+
+    def test_readonly_axis_blosc_to_sql(self):
+        # GH11880
+        if not _BLOSC_INSTALLED:
+            raise nose.SkipTest('no blosc')
+        if not self._SQLALCHEMY_INSTALLED:
+            raise nose.SkipTest('no sqlalchemy')
+        expected = DataFrame({'A': list('abcd')})
+        df = self.encode_decode(expected, compress='blosc')
+        eng = self._create_sql_engine("sqlite:///:memory:")
+        df.to_sql('test', eng, if_exists='append')
+        result = pandas.read_sql_table('test', eng, index_col='index')
+        result.index.names = [None]
+        assert_frame_equal(expected, result)
+
+    def test_readonly_axis_zlib_to_sql(self):
+        # GH11880
+        if not _ZLIB_INSTALLED:
+            raise nose.SkipTest('no zlib')
+        if not self._SQLALCHEMY_INSTALLED:
+            raise nose.SkipTest('no sqlalchemy')
+        expected = DataFrame({'A': list('abcd')})
+        df = self.encode_decode(expected, compress='zlib')
+        eng = self._create_sql_engine("sqlite:///:memory:")
+        df.to_sql('test', eng, if_exists='append')
+        result = pandas.read_sql_table('test', eng, index_col='index')
+        result.index.names = [None]
+        assert_frame_equal(expected, result)
+
 
 class TestEncoding(TestPackers):
         def setUp(self):