Repository: mne-tools/mne-python

Issue:
BUG: Error in handling fiff tags with "bad" text encoding
It sees that there is a small problem in decoding experimenter name tag from raw fiff files at least with the latest mne-python, as I remember succeeding some months ago with files from same dataset (with latest mne-python back then). It seems that my full name with the scandinavian letter "ä" has been used in user account creation for the MEG acquisition computer (personally I would have simply used "a" but I guess the admin had a Finnish keyboard). This letter, however, seems to be stored in non-utf-8 format (ISO-8859-1/Latin-1 is my best guess, or simply broken encoding). The end results is now that raw fiff file loading fails.

Small code change in fiff/tag.py (line 347) to simply

```
tag.data = str(tag.data.tostring().decode(encoding='UTF-8'))
```

is not enough, as it results an error "UnicodeDecodeError: 'utf8' codec can't decode byte 0xe4 in position 13: invalid continuation byte". Similar error comes up with forcing 'ISO-8859-1' encoding, so I guess the encoding is simply broken. However, for initial workaround solution, changing the line to e.g.

```
tag.data = str(tag.data.tostring().decode(encoding='UTF-8',errors='ignore'))
```

seems to work fine.

And yes I know  _could_ attempt the change the relevant code for good (if that minor change shows no side effects), but unfortunately I'm still learning the basics of python (i.e. only running slightly modified MNE-python code examples) AND github (i.e., created an account just for this :) and I have no idea what kind of encoding the raw fiff file should use in string tags..

Full traceback in case of problematic file

```
In [132]: raw = fiff.Raw(raw_fname)
Opening raw data file /datadir/megraw_mc/fs1a_raw_mc_tsss.fif...
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
/datadir/results/tmp/<ipython-input-132-233f57df4bce> in <module>()
----> 1 raw = fiff.Raw(raw_fname)

/virtualenv/mne_env1/src/mne/mne/fiff/raw.py in __init__(self, fnames, allow_maxshield, preload, proj, compensation, add_eeg_ref, verbose) 
/virtualenv/mne_env1/src/mne/mne/utils.py in verbose(function, *args, **kwargs)
    385         return ret
    386     else:
--> 387         ret = function(*args, **kwargs)
    388         return ret
    389

/virtualenv/mne_env1/src/mne/mne/fiff/raw.py in __init__(self, fnames, allow_maxshield, preload, proj, compensation, add_eeg_ref, verbose)  
     94
     95         raws = [self._read_raw_file(fname, allow_maxshield, preload,
---> 96                                     compensation) for fname in fnames]
     97         _check_raw_compatibility(raws)
     98

/virtualenv/mne_env1/src/mne/mne/fiff/raw.py in _read_raw_file(self, fname, allow_maxshield, preload, compensation, verbose)

/virtualenv/mne_env1/src/mne/mne/utils.py in verbose(function, *args, **kwargs)
    385         return ret
    386     else:
--> 387         ret = function(*args, **kwargs)
    388         return ret
    389

/virtualenv/mne_env1/src/mne/mne/fiff/raw.py in _read_raw_file(self, fname, allow_maxshield, preload, compensation, verbose)
    180
    181         #   Read the measurement info

--> 182         info, meas = read_meas_info(fid, tree)
    183
    184         #   Locate the data of interest


/virtualenv/mne_env1/src/mne/mne/fiff/meas_info.py in read_meas_info(fid, tree, verbose)

/virtualenv/mne_env1/src/mne/mne/utils.py in verbose(function, *args, **kwargs)
    385         return ret
    386     else:
--> 387         ret = function(*args, **kwargs)
    388         return ret
    389

/virtualenv/mne_env1/src/mne/mne/fiff/meas_info.py in read_meas_info(fid, tree, verbose)
    246                 ctf_head_t = cand
    247         elif kind == FIFF.FIFF_EXPERIMENTER:
--> 248             tag = read_tag(fid, pos)
    249             experimenter = tag.data
    250         elif kind == FIFF.FIFF_DESCRIPTION:

/virtualenv/mne_env1/src/mne/mne/fiff/tag.py in read_tag(fid, pos, shape, rlims)
    345                                             shape=shape, rlims=rlims)
    346                 # Use unicode or bytes depending on Py2/3

--> 347                 tag.data = str(tag.data.tostring().decode())
    348             elif tag.type == FIFF.FIFFT_DAU_PACK16:
    349                 tag.data = _fromstring_rows(fid, tag.size, dtype=">i2",

UnicodeDecodeError: 'ascii' codec can't decode byte 0xe4 in position 13: ordinal not in range(128)
```



Discussion:
Thanks for reporting @jkauramaki 
I think it should be straight forward to add a test that adds UTF-8 names on runtime and saves a temp file to reproduce and ultimately tackle this issue.

I can tackle this next week sometime if nobody else wants to.


Resolving patch:
diff --git a/mne/fiff/tag.py b/mne/fiff/tag.py
--- a/mne/fiff/tag.py
+++ b/mne/fiff/tag.py
@@ -4,11 +4,12 @@
 # License: BSD (3-clause)
 
 import struct
-import numpy as np
-from scipy import linalg
 import os
 import gzip
-from ..externals.six import b
+import sys
+import numpy as np
+from scipy import linalg
+from ..externals.six import b, text_type
 
 from .constants import FIFF
 
@@ -343,8 +344,10 @@ def read_tag(fid, pos=None, shape=None, rlims=None):
             elif tag.type == FIFF.FIFFT_STRING:
                 tag.data = _fromstring_rows(fid, tag.size, dtype=">c",
                                             shape=shape, rlims=rlims)
-                # Use unicode or bytes depending on Py2/3
-                tag.data = str(tag.data.tostring().decode())
+
+                # Always decode to unicode.
+                tag.data = text_type(tag.data.tostring().decode('utf-8'))
+
             elif tag.type == FIFF.FIFFT_DAU_PACK16:
                 tag.data = _fromstring_rows(fid, tag.size, dtype=">i2",
                                             shape=shape, rlims=rlims)
diff --git a/mne/fiff/write.py b/mne/fiff/write.py
--- a/mne/fiff/write.py
+++ b/mne/fiff/write.py
@@ -11,6 +11,7 @@
 import gzip
 import re
 import uuid
+import sys
 
 from .constants import FIFF
 from ..utils import logger
@@ -19,8 +20,10 @@
 def _write(fid, data, kind, data_size, FIFFT_TYPE, dtype):
     if isinstance(data, np.ndarray):
         data_size *= data.size
-    if isinstance(data, str):
-        data_size *= len(data)
+
+    # XXX for string types the data size is used as
+    # computed in ``write_string``.
+
     fid.write(np.array(kind, dtype='>i4').tostring())
     fid.write(np.array(FIFFT_TYPE, dtype='>i4').tostring())
     fid.write(np.array(data_size, dtype='>i4').tostring())
@@ -72,8 +75,11 @@ def write_complex128(fid, kind, data):
 
 def write_string(fid, kind, data):
     """Writes a string tag"""
-    data_size = 1
-    _write(fid, str(data), kind, data_size, FIFF.FIFFT_STRING, '>c')
+
+    str_data = data.encode('utf-8')  # Use unicode or bytes depending on Py2/3
+    data_size = len(str_data)  # therefore compute size here
+    my_dtype = '>a'  # py2/3 compatible on writing -- don't ask me why
+    _write(fid, str_data, kind, data_size, FIFF.FIFFT_STRING, my_dtype)
 
 
 def write_name_list(fid, kind, data):


Test patch:
diff --git a/mne/fiff/tests/test_raw.py b/mne/fiff/tests/test_raw.py
--- a/mne/fiff/tests/test_raw.py
+++ b/mne/fiff/tests/test_raw.py
@@ -1,4 +1,5 @@
 from __future__ import print_function
+
 # Author: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
 #         Denis Engemann <d.engemann@fz-juelich.de>
 #
@@ -8,6 +9,7 @@
 import os.path as op
 from copy import deepcopy
 import warnings
+import sys
 
 import numpy as np
 from numpy.testing import (assert_array_almost_equal, assert_array_equal,
@@ -19,6 +21,7 @@
 from mne import concatenate_events, find_events
 from mne.utils import (_TempDir, requires_nitime, requires_pandas,
                        requires_mne, run_subprocess)
+from mne.externals.six import text_type, b
 
 warnings.simplefilter('always')  # enable b/c these tests throw warnings
 
@@ -262,6 +265,16 @@ def test_load_bad_channels():
 def test_io_raw():
     """Test IO for raw data (Neuromag + CTF + gz)
     """
+    # test unicode io
+    for chars in [b'\xc3\xa4\xc3\xb6\xc3\xa9', b'a']:
+        with Raw(fif_fname) as r:
+            desc1 = r.info['description'] = chars.decode('utf-8')
+            temp_file = op.join(tempdir, 'raw.fif')
+            r.save(temp_file, overwrite=True)
+            with Raw(temp_file) as r2:
+                desc2 = r2.info['description']
+            assert_equal(desc1, desc2)
+
     # Let's construct a simple test for IO first
     raw = Raw(fif_fname, preload=True)
     raw.crop(0, 3.5)