Repository: nipy/nipype

Issue:
python 2 bytes/str issue
in `CommandLine` interface it `terminal_output` is `allatonce`, we do the following:

https://github.com/nipy/nipype/blob/master/nipype/interfaces/base.py#L1257

however on python 2, when the underlying command generates an error the returned string (at least on centos 7), contains some unsavory bytes. this results in an error as seen below 

one solution is to detect whether we are on python 2 and not decode the string.  any ideas how to solve this for both platforms simultaneously? @matthew-brett @bcipolli 

```
In [45]: from subprocess import Popen, PIPE

In [46]: proc = Popen('cp foo goo', shell=True, stdout=PIPE, stderr=PIPE)

In [47]: stdout, stderr = proc.communicate()

In [48]: stderr
Out[48]: 'cp: cannot stat \xe2\x80\x98foo\xe2\x80\x99: No such file or directory\n'

In [49]: str(stderr)
Out[49]: 'cp: cannot stat \xe2\x80\x98foo\xe2\x80\x99: No such file or directory\n'

In [50]: stderr.decode()
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-50-80fc7c733b30> in <module>()
----> 1 stderr.decode()

UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 16: ordinal not in range(128)

In [51]: stderr.decode('ISO-8859-1')
Out[51]: u'cp: cannot stat \xe2\x80\x98foo\xe2\x80\x99: No such file or directory\n'

In [53]: str(stderr.decode('ISO-8859-1'))
---------------------------------------------------------------------------
UnicodeEncodeError                        Traceback (most recent call last)
<ipython-input-53-e11a9dfd7944> in <module>()
----> 1 str(stderr.decode('ISO-8859-1'))

UnicodeEncodeError: 'ascii' codec can't encode characters in position 16-18: ordinal not in range(128)
```



Discussion:
That's weird - I don't get that on OSX.  How about using a list instead of string for the arguments - do you get the same error?

Testing on my Centos 7 system, this is because of the fancy open / close quotes around the filename in the error (same when running the command from bash).  I guess the system is returning the string using its locale settings:

```
>>> locale.getdefaultlocale()[1]
'UTF-8'
```

So I guess you should do something like:

```
stderr.decode(locale.getdefaultlocale()[1])
```

on Pythons 2 or 3, to get the right error message.


Resolving patch:
diff --git a/nipype/interfaces/base.py b/nipype/interfaces/base.py
--- a/nipype/interfaces/base.py
+++ b/nipype/interfaces/base.py
@@ -19,6 +19,7 @@
 from copy import deepcopy
 import datetime
 import errno
+import locale
 import os
 import re
 import platform
@@ -47,7 +48,7 @@
 from ..utils.provenance import write_provenance
 from .. import config, logging, LooseVersion
 from .. import __version__
-from ..external.six import string_types
+from ..external.six import string_types, text_type
 
 nipype_version = LooseVersion(__version__)
 
@@ -1205,8 +1206,8 @@ def run_command(runtime, output=None, timeout=0.01, redirect_x=False):
     if output == 'file':
         errfile = os.path.join(runtime.cwd, 'stderr.nipype')
         outfile = os.path.join(runtime.cwd, 'stdout.nipype')
-        stderr = open(errfile, 'wt')  # t=='text'===default
-        stdout = open(outfile, 'wt')
+        stderr = open(errfile, 'wb')  # t=='text'===default
+        stdout = open(outfile, 'wb')
 
         proc = subprocess.Popen(cmdline,
                                 stdout=stdout,
@@ -1256,26 +1257,17 @@ def _process(drain=0):
         result['merged'] = [r[1] for r in temp]
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        if stdout and isinstance(stdout, bytes):
-            try:
-                stdout = stdout.decode()
-            except UnicodeDecodeError:
-                stdout = stdout.decode("ISO-8859-1")
-        if stderr and isinstance(stderr, bytes):
-            try:
-                stderr = stderr.decode()
-            except UnicodeDecodeError:
-                stderr = stderr.decode("ISO-8859-1")
-
-        result['stdout'] = str(stdout).split('\n')
-        result['stderr'] = str(stderr).split('\n')
+        stdout = stdout.decode(locale.getdefaultlocale()[1])
+        stderr = stderr.decode(locale.getdefaultlocale()[1])
+        result['stdout'] = stdout.split('\n')
+        result['stderr'] = stderr.split('\n')
         result['merged'] = ''
     if output == 'file':
         ret_code = proc.wait()
         stderr.flush()
         stdout.flush()
-        result['stdout'] = [line.strip() for line in open(outfile).readlines()]
-        result['stderr'] = [line.strip() for line in open(errfile).readlines()]
+        result['stdout'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(outfile, 'rb').readlines()]
+        result['stderr'] = [line.decode(locale.getdefaultlocale()[1]).strip() for line in open(errfile, 'rb').readlines()]
         result['merged'] = ''
     if output == 'none':
         proc.communicate()


Test patch:
diff --git a/nipype/interfaces/spm/tests/test_base.py b/nipype/interfaces/spm/tests/test_base.py
--- a/nipype/interfaces/spm/tests/test_base.py
+++ b/nipype/interfaces/spm/tests/test_base.py
@@ -12,7 +12,7 @@
 from nipype.interfaces.spm import no_spm
 import nipype.interfaces.matlab as mlab
 from nipype.interfaces.spm.base import SPMCommandInputSpec
-from nipype.interfaces.base import traits
+from nipype.interfaces.base import traits, text_type
 
 try:
     matlab_cmd = os.environ['MATLABCMD']
@@ -57,7 +57,7 @@ def test_scan_for_fnames():
     def test_spm_path():
         spm_path = spm.Info.version()['path']
         if spm_path is not None:
-            yield assert_equal, type(spm_path), type('')
+            yield assert_equal, type(spm_path), text_type
             yield assert_true, 'spm' in spm_path
 
 
diff --git a/nipype/interfaces/tests/test_base.py b/nipype/interfaces/tests/test_base.py
--- a/nipype/interfaces/tests/test_base.py
+++ b/nipype/interfaces/tests/test_base.py
@@ -14,7 +14,7 @@
                             skipif)
 import nipype.interfaces.base as nib
 from nipype.utils.filemanip import split_filename
-from nipype.interfaces.base import Undefined, config
+from nipype.interfaces.base import Undefined, config, text_type
 from traits.testing.nose_tools import skip
 import traits.api as traits
 
@@ -661,7 +661,7 @@ def test_CommandLine_output():
     ci.inputs.terminal_output = 'file'
     res = ci.run()
     yield assert_true, 'stdout.nipype' in res.runtime.stdout
-    yield assert_equal, type(res.runtime.stdout), type('hi')
+    yield assert_equal, type(res.runtime.stdout), text_type
     ci = nib.CommandLine(command='ls -l')
     ci.inputs.terminal_output = 'none'
     res = ci.run()