Repository: nipy/nipype

Issue:
UnicodeDecodeError: 'utf-8' codec can't decode in run_command
Running FMRIPREP using singularity.

```
Node: _autorecon30
Working directory: /scratch/03763/oesteban/fmriprep-phase2/work/ds000172/sub-control01/fmriprep_wf/single_subject_control01_wf/anat_preproc_wf/surface_recon_wf/autorecon_resume_wf/autorecon3/mapflow/_autorecon
30

Node inputs:

T1_files = <undefined>
T2_file = <undefined>
args = <undefined>
big_ventricles = <undefined>
brainstem = <undefined>
directive = autorecon3
environ = {}
expert = <undefined>
flags = <undefined>
hemi = lh
hippocampal_subfields_T1 = <undefined>
hippocampal_subfields_T2 = <undefined>
hires = <undefined>
ignore_exception = False
mprage = <undefined>
mri_aparc2aseg = <undefined>
mri_ca_label = <undefined>
mri_ca_normalize = <undefined>
mri_ca_register = <undefined>
mri_edit_wm_with_aseg = <undefined>
mri_em_register = <undefined>
mri_fill = <undefined>
mri_mask = <undefined>
mri_normalize = <undefined>
mri_pretess = <undefined>
mri_remove_neck = <undefined>
mri_segment = <undefined>
mri_segstats = <undefined>
mri_tessellate = <undefined>
mri_watershed = <undefined>
mris_anatomical_stats = <undefined>
mris_ca_label = <undefined>
mris_fix_topology = <undefined>
mris_inflate = <undefined>
mris_make_surfaces = <undefined>
mris_register = <undefined>
mris_smooth = <undefined>
mris_sphere = <undefined>
mris_surf2vol = <undefined>
mrisp_paint = <undefined>
openmp = 8
parallel = <undefined>
subject_id = sub-control01
subjects_dir = /work/03843/crn_plab/stampede2/fmriprep-phase2/derivatives/ds000172/freesurfer
talairach = <undefined>
terminal_output = <undefined>
use_T2 = False
xopts = <undefined>

Traceback (most recent call last):
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/plugins/multiproc.py", line 51, in run_node
    result['result'] = node.run(updatehash=updatehash)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 407, in run
    self._run_interface()
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 517, in _run_interface
    self._result = self._run_command(execute)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 650, in _run_command
    result = self._interface.run()
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1088, in run
    runtime = self._run_interface(runtime)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1654, in _run_interface
    runtime = run_command(runtime, output=self.terminal_output)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1423, in run_command
    result['stdout'] = stdout.decode(default_encoding).split('\n')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc7 in position 9789: invalid continuation byte
```


Discussion:
default_encoding:
```
nid00024(3)$ singularity exec /work/03843/crn_plab/singularity/poldracklab_fmriprep_1.0.0-rc7-747-2017-10-17-cc920f8dcef8.img python -c "import sys; print(sys.getdefaultencoding())"
utf-8
```
(and from sys)
```
nid00024(4)$ singularity exec /work/03843/crn_plab/singularity/poldracklab_fmriprep_1.0.0-rc7-747-2017-10-17-cc920f8dcef8.img python -c "import locale; print(locale.getdefaultlocale()[1])"
UTF-8
```
We could use `chardet` as a fallback for these decodings:

```
python -c "import chardet; print(chardet.detect(bytearray([0xc7])));"
{'encoding': 'ISO-8859-1', 'confidence': 0.73, 'language': ''}
```
@oesteban - do you know where these characters came from?
Running FreeSurfer's autorecon, but no idea why that character should be there. Does not make much sense. However, a corrupted output is better than no output at all right?
yes - decode has an option to take characters that it didn't read and replace them with some properly encoded character. 

@yarikoptic - we did this for datalad, do you have a quick pointer?
that'd be sweet

Resolving patch:
diff --git a/nipype/interfaces/base.py b/nipype/interfaces/base.py
--- a/nipype/interfaces/base.py
+++ b/nipype/interfaces/base.py
@@ -38,7 +38,7 @@
 from ..utils.provenance import write_provenance
 from ..utils.misc import is_container, trim, str2bool
 from ..utils.filemanip import (md5, hash_infile, FileNotFoundError, hash_timestamp,
-                               split_filename, to_str)
+                               split_filename, to_str, read_stream)
 from .traits_extension import (
     traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
     File, Directory, DictStrStr, has_metadata, ImageFile)
@@ -1268,9 +1268,7 @@ def __init__(self, name, impl):
         self._buf = ''
         self._rows = []
         self._lastidx = 0
-        self.default_encoding = locale.getdefaultlocale()[1]
-        if self.default_encoding is None:
-            self.default_encoding = 'UTF-8'
+        self.default_encoding = locale.getdefaultlocale()[1] or 'UTF-8'
 
     def fileno(self):
         "Pass-through for file descriptor."
@@ -1349,10 +1347,6 @@ def run_command(runtime, output=None, timeout=0.01):
     cmdline = runtime.cmdline
     env = _canonicalize_env(runtime.environ)
 
-    default_encoding = locale.getdefaultlocale()[1]
-    if default_encoding is None:
-        default_encoding = 'UTF-8'
-
     errfile = None
     outfile = None
     stdout = sp.PIPE
@@ -1420,19 +1414,22 @@ def _process(drain=0):
 
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        result['stdout'] = stdout.decode(default_encoding).split('\n')
-        result['stderr'] = stderr.decode(default_encoding).split('\n')
+        result['stdout'] = read_stream(stdout, logger=iflogger)
+        result['stderr'] = read_stream(stderr, logger=iflogger)
 
     elif output.startswith('file'):
         proc.wait()
         if outfile is not None:
             stdout.flush()
-            result['stdout'] = [line.decode(default_encoding).strip()
-                                for line in open(outfile, 'rb').readlines()]
+            with open(outfile, 'rb') as ofh:
+                stdoutstr = ofh.read()
+            result['stdout'] = read_stream(stdoutstr, logger=iflogger)
+
         if errfile is not None:
             stderr.flush()
-            result['stderr'] = [line.decode(default_encoding).strip()
-                                for line in open(errfile, 'rb').readlines()]
+            with open(errfile, 'rb') as efh:
+                stderrstr = efh.read()
+            result['stderr'] = read_stream(stderrstr, logger=iflogger)
 
         if output == 'file':
             result['merged'] = result['stdout']
diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py
--- a/nipype/utils/filemanip.py
+++ b/nipype/utils/filemanip.py
@@ -5,16 +5,13 @@
 
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
-from builtins import str, bytes, open
-
-from future import standard_library
-standard_library.install_aliases()
 
 import sys
 import pickle
 import subprocess
 import gzip
 import hashlib
+import locale
 from hashlib import md5
 import os
 import re
@@ -23,10 +20,15 @@
 import simplejson as json
 import numpy as np
 
+from builtins import str, bytes, open
+
 from .. import logging, config
 from .misc import is_container
 from ..interfaces.traits_extension import isdefined
 
+from future import standard_library
+standard_library.install_aliases()
+
 fmlogger = logging.getLogger('utils')
 
 
@@ -596,6 +598,26 @@ def crash2txt(filename, record):
         fp.write(''.join(record['traceback']))
 
 
+def read_stream(stream, logger=None, encoding=None):
+    """
+    Robustly reads a stream, sending a warning to a logger
+    if some decoding error was raised.
+
+    >>> read_stream(bytearray([65, 0xc7, 65, 10, 66]))  # doctest: +ELLIPSIS +ALLOW_UNICODE
+    ['A...A', 'B']
+
+
+    """
+    default_encoding = encoding or locale.getdefaultlocale()[1] or 'UTF-8'
+    logger = logger or fmlogger
+    try:
+        out = stream.decode(default_encoding)
+    except UnicodeDecodeError as err:
+        out = stream.decode(default_encoding, errors='replace')
+        logger.warning('Error decoding string: %s', err)
+    return out.splitlines()
+
+
 def savepkl(filename, record):
     if filename.endswith('pklz'):
         pkl_file = gzip.open(filename, 'wb')


Test patch: