Repository: nipy/nipype

Issue:
UnicodeDecodeError: 'utf-8' codec can't decode in run_command
Running FMRIPREP using singularity.

```
Node: _autorecon30
Working directory: /scratch/03763/oesteban/fmriprep-phase2/work/ds000172/sub-control01/fmriprep_wf/single_subject_control01_wf/anat_preproc_wf/surface_recon_wf/autorecon_resume_wf/autorecon3/mapflow/_autorecon
30

Node inputs:

T1_files = <undefined>
T2_file = <undefined>
args = <undefined>
big_ventricles = <undefined>
brainstem = <undefined>
directive = autorecon3
environ = {}
expert = <undefined>
flags = <undefined>
hemi = lh
hippocampal_subfields_T1 = <undefined>
hippocampal_subfields_T2 = <undefined>
hires = <undefined>
ignore_exception = False
mprage = <undefined>
mri_aparc2aseg = <undefined>
mri_ca_label = <undefined>
mri_ca_normalize = <undefined>
mri_ca_register = <undefined>
mri_edit_wm_with_aseg = <undefined>
mri_em_register = <undefined>
mri_fill = <undefined>
mri_mask = <undefined>
mri_normalize = <undefined>
mri_pretess = <undefined>
mri_remove_neck = <undefined>
mri_segment = <undefined>
mri_segstats = <undefined>
mri_tessellate = <undefined>
mri_watershed = <undefined>
mris_anatomical_stats = <undefined>
mris_ca_label = <undefined>
mris_fix_topology = <undefined>
mris_inflate = <undefined>
mris_make_surfaces = <undefined>
mris_register = <undefined>
mris_smooth = <undefined>
mris_sphere = <undefined>
mris_surf2vol = <undefined>
mrisp_paint = <undefined>
openmp = 8
parallel = <undefined>
subject_id = sub-control01
subjects_dir = /work/03843/crn_plab/stampede2/fmriprep-phase2/derivatives/ds000172/freesurfer
talairach = <undefined>
terminal_output = <undefined>
use_T2 = False
xopts = <undefined>

Traceback (most recent call last):
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/plugins/multiproc.py", line 51, in run_node
    result['result'] = node.run(updatehash=updatehash)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 407, in run
    self._run_interface()
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 517, in _run_interface
    self._result = self._run_command(execute)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/pipeline/engine/nodes.py", line 650, in _run_command
    result = self._interface.run()
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1088, in run
    runtime = self._run_interface(runtime)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1654, in _run_interface
    runtime = run_command(runtime, output=self.terminal_output)
  File "/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/nipype/interfaces/base.py", line 1423, in run_command
    result['stdout'] = stdout.decode(default_encoding).split('\n')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc7 in position 9789: invalid continuation byte
```

Comments:

Patch:
diff --git a/nipype/interfaces/base.py b/nipype/interfaces/base.py
index 79812d4b1..f8e845d94 100644
--- a/nipype/interfaces/base.py
+++ b/nipype/interfaces/base.py
@@ -38,7 +38,7 @@ from .. import config, logging, LooseVersion, __version__
 from ..utils.provenance import write_provenance
 from ..utils.misc import is_container, trim, str2bool
 from ..utils.filemanip import (md5, hash_infile, FileNotFoundError, hash_timestamp,
-                               split_filename, to_str)
+                               split_filename, to_str, read_stream)
 from .traits_extension import (
     traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
     File, Directory, DictStrStr, has_metadata, ImageFile)
@@ -1268,9 +1268,7 @@ class Stream(object):
         self._buf = ''
         self._rows = []
         self._lastidx = 0
-        self.default_encoding = locale.getdefaultlocale()[1]
-        if self.default_encoding is None:
-            self.default_encoding = 'UTF-8'
+        self.default_encoding = locale.getdefaultlocale()[1] or 'UTF-8'
 
     def fileno(self):
         "Pass-through for file descriptor."
@@ -1349,10 +1347,6 @@ def run_command(runtime, output=None, timeout=0.01):
     cmdline = runtime.cmdline
     env = _canonicalize_env(runtime.environ)
 
-    default_encoding = locale.getdefaultlocale()[1]
-    if default_encoding is None:
-        default_encoding = 'UTF-8'
-
     errfile = None
     outfile = None
     stdout = sp.PIPE
@@ -1420,19 +1414,22 @@ def run_command(runtime, output=None, timeout=0.01):
 
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        result['stdout'] = stdout.decode(default_encoding).split('\n')
-        result['stderr'] = stderr.decode(default_encoding).split('\n')
+        result['stdout'] = read_stream(stdout, logger=iflogger)
+        result['stderr'] = read_stream(stderr, logger=iflogger)
 
     elif output.startswith('file'):
         proc.wait()
         if outfile is not None:
             stdout.flush()
-            result['stdout'] = [line.decode(default_encoding).strip()
-                                for line in open(outfile, 'rb').readlines()]
+            with open(outfile, 'rb') as ofh:
+                stdoutstr = ofh.read()
+            result['stdout'] = read_stream(stdoutstr, logger=iflogger)
+
         if errfile is not None:
             stderr.flush()
-            result['stderr'] = [line.decode(default_encoding).strip()
-                                for line in open(errfile, 'rb').readlines()]
+            with open(errfile, 'rb') as efh:
+                stderrstr = efh.read()
+            result['stderr'] = read_stream(stderrstr, logger=iflogger)
 
         if output == 'file':
             result['merged'] = result['stdout']
diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py
index 0138062e5..5f39f1cc9 100644
--- a/nipype/interfaces/freesurfer/preprocess.py
+++ b/nipype/interfaces/freesurfer/preprocess.py
@@ -2387,8 +2387,8 @@ class ConcatenateLTAInputSpec(FSTraitedSpec):
         File(exists=True), 'identity.nofile', argstr='%s', position=-2,
         mandatory=True, desc='maps dst1(src2) to dst2')
     out_file = File(
-        'concat.lta', usedefault=True, position=-1, argstr='%s',
-        hash_files=False,
+        position=-1, argstr='%s', hash_files=False, name_source=['in_lta1'],
+        name_template='%s_concat', keep_extension=True,
         desc='the combined LTA maps: src1 to dst2 = LTA2*LTA1')
 
     # Inversion and transform type
@@ -2434,7 +2434,7 @@ class ConcatenateLTA(FSCommand):
     >>> conc_lta.inputs.in_lta1 = 'lta1.lta'
     >>> conc_lta.inputs.in_lta2 = 'lta2.lta'
     >>> conc_lta.cmdline # doctest: +ALLOW_UNICODE
-    'mri_concatenate_lta lta1.lta lta2.lta concat.lta'
+    'mri_concatenate_lta lta1.lta lta2.lta lta1_concat.lta'
 
     You can use 'identity.nofile' as the filename for in_lta2, e.g.:
 
@@ -2459,8 +2459,3 @@ class ConcatenateLTA(FSCommand):
         if name == 'out_type':
             value = {'VOX2VOX': 0, 'RAS2RAS': 1}[value]
         return super(ConcatenateLTA, self)._format_arg(name, spec, value)
-
-    def _list_outputs(self):
-        outputs = self.output_spec().get()
-        outputs['out_file'] = os.path.abspath(self.inputs.out_file)
-        return outputs
diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py
index e321a597a..72cb1fc15 100644
--- a/nipype/utils/filemanip.py
+++ b/nipype/utils/filemanip.py
@@ -5,16 +5,13 @@
 
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
-from builtins import str, bytes, open
-
-from future import standard_library
-standard_library.install_aliases()
 
 import sys
 import pickle
 import subprocess
 import gzip
 import hashlib
+import locale
 from hashlib import md5
 import os
 import re
@@ -23,10 +20,15 @@ import posixpath
 import simplejson as json
 import numpy as np
 
+from builtins import str, bytes, open
+
 from .. import logging, config
 from .misc import is_container
 from ..interfaces.traits_extension import isdefined
 
+from future import standard_library
+standard_library.install_aliases()
+
 fmlogger = logging.getLogger('utils')
 
 
@@ -596,6 +598,26 @@ def crash2txt(filename, record):
         fp.write(''.join(record['traceback']))
 
 
+def read_stream(stream, logger=None, encoding=None):
+    """
+    Robustly reads a stream, sending a warning to a logger
+    if some decoding error was raised.
+
+    >>> read_stream(bytearray([65, 0xc7, 65, 10, 66]))  # doctest: +ELLIPSIS +ALLOW_UNICODE
+    ['A...A', 'B']
+
+
+    """
+    default_encoding = encoding or locale.getdefaultlocale()[1] or 'UTF-8'
+    logger = logger or fmlogger
+    try:
+        out = stream.decode(default_encoding)
+    except UnicodeDecodeError as err:
+        out = stream.decode(default_encoding, errors='replace')
+        logger.warning('Error decoding string: %s', err)
+    return out.splitlines()
+
+
 def savepkl(filename, record):
     if filename.endswith('pklz'):
         pkl_file = gzip.open(filename, 'wb')


Test patch:
diff --git a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py
index 1f957d35f..2357c8709 100644
--- a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py
+++ b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py
@@ -28,8 +28,10 @@ def test_ConcatenateLTA_inputs():
     ),
     out_file=dict(argstr='%s',
     hash_files=False,
+    keep_extension=True,
+    name_source=['in_lta1'],
+    name_template='%s_concat',
     position=-1,
-    usedefault=True,
     ),
     out_type=dict(argstr='-out_type %d',
     ),