Repository: pandas-dev/pandas

Issue:
ValueError when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2
#### Code Sample, a copy-pastable example if possible

```python
# Part of the code to be executed in Python 2
import pandas as pd
df = pd.DataFrame([[1, 2, 3, "D"]],
                  columns=['A', 'B', 'C', 'D'],
                  index=pd.Index(['ABC'], name='INDEX_NAME'))

store = pd.HDFStore("test.hdf", mode='w')
store.put(value=df, key="df", format='fixed')
store.close()

# Part of the code to be executed in Python 3
import pandas as pd
store = pd.HDFStore('test.hdf', mode='r')
df = store['df']
```
#### Problem description

When I run the above code (The first part with a Python 2 environment and the second part with a Python 3, see details below) , I get the following exception:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-7-30b5710694c8> in <module>
----> 1 df = store['df']

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in __getitem__(self, key)
    505 
    506     def __getitem__(self, key):
--> 507         return self.get(key)
    508 
    509     def __setitem__(self, key, value):

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in get(self, key)
    693         if group is None:
    694             raise KeyError('No object named %s in the file' % key)
--> 695         return self._read_group(group)
    696 
    697     def select(self, key, where=None, start=None, stop=None, columns=None,

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in _read_group(self, group, **kwargs)
   1373         s = self._create_storer(group)
   1374         s.infer_axes()
-> 1375         return s.read(**kwargs)
   1376 
   1377 

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in read(self, start, stop, **kwargs)
   2926 
   2927             _start, _stop = (start, stop) if i == select_axis else (None, None)
-> 2928             ax = self.read_index('axis%d' % i, start=_start, stop=_stop)
   2929             axes.append(ax)
   2930 

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in read_index(self, key, **kwargs)
   2521             return self.read_sparse_intindex(key, **kwargs)
   2522         elif variety == u('regular'):
-> 2523             _, index = self.read_index_node(getattr(self.group, key), **kwargs)
   2524             return index
   2525         else:  # pragma: no cover

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in read_index_node(self, node, start, stop)
   2651             index = factory(_unconvert_index(data, kind,
   2652                                              encoding=self.encoding,
-> 2653                                              errors=self.errors), **kwargs)
   2654 
   2655         index.name = name

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in _unconvert_index(data, kind, encoding, errors)
   4561     elif kind in (u('string')):
   4562         index = _unconvert_string_array(data, nan_rep=None, encoding=encoding,
-> 4563                                         errors=errors)
   4564     elif kind == u('object'):
   4565         index = np.asarray(data[0])

~/miniconda3/envs/dev-env-py37/lib/python3.6/site-packages/pandas/io/pytables.py in _unconvert_string_array(data, nan_rep, encoding, errors)
   4654         nan_rep = 'nan'
   4655 
-> 4656     data = libwriters.string_array_replace_from_nan_rep(data, nan_rep)
   4657     return data.reshape(shape)
   4658 

pandas/_libs/writers.pyx in pandas._libs.writers.string_array_replace_from_nan_rep()

ValueError: Buffer dtype mismatch, expected 'Python object' but got 'double'
```

This exception is raised only in 'fixed' format, no issues with table format (The DataFrame is read correctly). 


#### Expected Output

The expected output is the same DataFrame as the one wrote in Python 2.

#### Output of ``pd.show_versions()``

<details>

[Python 2]

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.15.final.0
python-bits: 64
OS: Linux
OS-release: 4.4.59+
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.23.4
pytest: None
pip: 18.1
setuptools: 40.6.3
Cython: None
numpy: 1.15.4
scipy: None
pyarrow: None
xarray: None
IPython: 5.8.0
sphinx: None
patsy: None
dateutil: 2.7.5
pytz: 2018.7
blosc: None
bottleneck: None
tables: 3.4.4
numexpr: 2.6.8
feather: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

[Python 3]

INSTALLED VERSIONS
------------------
commit: None
python: 3.6.7.final.0
python-bits: 64
OS: Linux
OS-release: 4.4.59+
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.23.4
pytest: None
pip: 18.1
setuptools: 40.6.3
Cython: None
numpy: 1.15.4
scipy: None
pyarrow: None
xarray: None
IPython: 7.2.0
sphinx: None
patsy: None
dateutil: 2.7.5
pytz: 2018.7
blosc: None
bottleneck: None
tables: 3.4.4
numexpr: 2.6.8
feather: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

</details>



Discussion:
cc @jreback 
we don't have a lot of guarantees around fixed format migration from py2 to py3. would have to have user investigation & a community PR to fix.

Resolving patch:
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -2501,7 +2501,7 @@ def set_attrs(self):
     def get_attrs(self):
         """ retrieve our attributes """
         self.encoding = _ensure_encoding(getattr(self.attrs, 'encoding', None))
-        self.errors = getattr(self.attrs, 'errors', 'strict')
+        self.errors = _ensure_decoded(getattr(self.attrs, 'errors', 'strict'))
         for n in self.attributes:
             setattr(self, n, _ensure_decoded(getattr(self.attrs, n, None)))
 
@@ -2661,6 +2661,7 @@ def read_index_node(self, node, start=None, stop=None):
 
         if 'name' in node._v_attrs:
             name = _ensure_str(node._v_attrs.name)
+            name = _ensure_decoded(name)
 
         index_class = self._alias_to_class(_ensure_decoded(
             getattr(node._v_attrs, 'index_class', '')))


Test patch:
diff --git a/pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5 b/pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5
new file mode 100644
Binary files /dev/null and b/pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5 differ
diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py
--- a/pandas/tests/io/test_pytables.py
+++ b/pandas/tests/io/test_pytables.py
@@ -4540,6 +4540,20 @@ def test_pytables_native2_read(self, datapath):
             d1 = store['detector']
             assert isinstance(d1, DataFrame)
 
+    def test_legacy_table_fixed_format_read_py2(self, datapath):
+        # GH 24510
+        # legacy table with fixed format written en Python 2
+        with ensure_clean_store(
+                datapath('io', 'data', 'legacy_hdf',
+                         'legacy_table_fixed_py2.h5'),
+                mode='r') as store:
+            result = store.select('df')
+            expected = pd.DataFrame([[1, 2, 3, 'D']],
+                                    columns=['A', 'B', 'C', 'D'],
+                                    index=pd.Index(['ABC'],
+                                                   name='INDEX_NAME'))
+            assert_frame_equal(expected, result)
+
     def test_legacy_table_read(self, datapath):
         # legacy table types
         with ensure_clean_store(