Repository: mne-tools/mne-python

Issue:
The current design of mne.bem.make_watershed_bem is incompatible with ipython
#### Describe the bug
Runinng `mne.bem.make_watershed_bem(fs_subject)` within ipython (actually, this was tried it in Jupyter notebook more specifically) crashes with:

```
---------------------------------------------------------------------------
UnsupportedOperation                      Traceback (most recent call last)
<ipython-input-8-a3517d055ccb> in <module>
----> 1 mne.bem.make_watershed_bem(fs_subject, overwrite=True)

</home/christian/mne-python/mne/externals/decorator.py:decorator-gen-50> in make_watershed_bem(subject, subjects_dir, overwrite, volume, atlas, gcaatlas, preflood, show, copy, T1, brainmask, verbose)

~/mne-python/mne/utils/_logging.py in wrapper(*args, **kwargs)
     88             with use_log_level(verbose_level):
     89                 return function(*args, **kwargs)
---> 90         return function(*args, **kwargs)
     91     return FunctionMaker.create(
     92         function, 'return decfunc(%(signature)s)',

~/mne-python/mne/bem.py in make_watershed_bem(subject, subjects_dir, overwrite, volume, atlas, gcaatlas, preflood, show, copy, T1, brainmask, verbose)
   1148                 % (ws_dir, ' '.join(cmd)))
   1149     os.makedirs(op.join(ws_dir))
-> 1150     run_subprocess_env(cmd)
   1151     if op.isfile(T1_mgz):
   1152         new_info = _extract_volume_info(T1_mgz)

</home/christian/mne-python/mne/externals/decorator.py:decorator-gen-1> in run_subprocess(command, return_code, verbose, *args, **kwargs)

~/mne-python/mne/utils/_logging.py in wrapper(*args, **kwargs)
     88             with use_log_level(verbose_level):
     89                 return function(*args, **kwargs)
---> 90         return function(*args, **kwargs)
     91     return FunctionMaker.create(
     92         function, 'return decfunc(%(signature)s)',

~/mne-python/mne/utils/misc.py in run_subprocess(command, return_code, verbose, *args, **kwargs)
    113         The return code, only returned if ``return_code == True``.
    114     """
--> 115     with running_subprocess(command, *args, **kwargs) as p:
    116         stdout_, stderr = p.communicate()
    117     stdout_ = u'' if stdout_ is None else stdout_.decode('utf-8')

~/anaconda3/lib/python3.7/contextlib.py in __enter__(self)
    110         del self.args, self.kwds, self.func
    111         try:
--> 112             return next(self.gen)
    113         except StopIteration:
    114             raise RuntimeError("generator didn't yield") from None

~/mne-python/mne/utils/misc.py in running_subprocess(command, after, verbose, *args, **kwargs)
    187     try:
    188         print(args, kwargs)
--> 189         p = subprocess.Popen(command, *args, **kwargs)
    190     except Exception:
    191         if isinstance(command, str):

~/anaconda3/lib/python3.7/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    726         (p2cread, p2cwrite,
    727          c2pread, c2pwrite,
--> 728          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
    729 
    730         # We wrap OS handles *before* launching the child, otherwise a

~/anaconda3/lib/python3.7/subprocess.py in _get_handles(self, stdin, stdout, stderr)
   1357             else:
   1358                 # Assuming file-like object
-> 1359                 c2pwrite = stdout.fileno()
   1360 
   1361             if stderr is None:

UnsupportedOperation: fileno
```

My undestanding of this issue is that the problems comes from `sys.stdout` being passed within the kwargs argument to subprocess.Popen() by `mne.utils.misc.running_subprocess`. Thus, `sys.stdout` gets passed as `sysout` to `Popen()` and the object contained in the variable `sys.stdout` does not have a fileno method: Within a Jupyter notebook:

```
import sys
print("#1:", sys.stdout)
print("#2:", sys.__stdout__)
print("#3:", sys.__stdout__.fileno())
print("#4:", sys.stdout.fileno())
```
gives the following output:
```
#1: <ipykernel.iostream.OutStream object at 0x7fc4e055fdd0>
#2: <_io.TextIOWrapper name='<stdout>' mode='w' encoding='UTF-8'>
#3: 1

---------------------------------------------------------------------------
UnsupportedOperation                      Traceback (most recent call last)
<ipython-input-19-ac77b7d7dd6f> in <module>
      3 print("#2:", sys.__stdout__)
      4 print("#3:", sys.__stdout__.fileno())
----> 5 print("#4:", sys.stdout.fileno())

UnsupportedOperation: fileno
```
Any mne functionnality relying on `mne.utils.misc.running_subprocess` is therefore expected to fail in a similar fashion.

Actually, I am sure you will be familiar with the issue since I found https://github.com/ipython/ipykernel/issues/272 related to this issue by @larsoner . Not sure if the fact that this causes bugs in MNE was spotted since I did not find corresponding issues reported for MNE. 

#### Additional information
Platform:      Linux-5.0.0-32-generic-x86_64-with-debian-buster-sid
Python:        3.7.4 (default, Aug 13 2019, 20:35:49)  [GCC 7.3.0]
Executable:    /home/christian/anaconda3/bin/python
CPU:           x86_64: 4 cores
Memory:        7.5 GB
mne:           0.20.dev0
numpy:         1.17.4 {blas=openblas, lapack=openblas}
scipy:         1.3.1
matplotlib:    3.1.1 {backend=module://ipykernel.pylab.backend_inline}
sklearn:       0.21.3
numba:         Not found
nibabel:       2.5.1
cupy:          Not found
pandas:        0.25.1
dipy:          Not found
mayavi:        4.7.1 {qt_api=pyqt5, PyQt5=5.9.2}
pyvista:       Not found
vtk:           8.1.2


Discussion:

Resolving patch:
diff --git a/mne/utils/misc.py b/mne/utils/misc.py
--- a/mne/utils/misc.py
+++ b/mne/utils/misc.py
@@ -7,8 +7,6 @@
 from contextlib import contextmanager
 import fnmatch
 import inspect
-from io import StringIO
-import logging
 from math import log
 import os
 from string import Formatter
@@ -112,11 +110,21 @@ def run_subprocess(command, return_code=False, verbose=None, *args, **kwargs):
     code : int
         The return code, only returned if ``return_code == True``.
     """
+    all_out = ''
+    all_err = ''
     with running_subprocess(command, *args, **kwargs) as p:
-        stdout_, stderr = p.communicate()
-    stdout_ = u'' if stdout_ is None else stdout_.decode('utf-8')
-    stderr = u'' if stderr is None else stderr.decode('utf-8')
-    output = (stdout_, stderr)
+        while True:
+            out = p.stdout.readline().decode('utf-8')
+            err = p.stderr.readline().decode('utf-8')
+            if out == '' and err == '' and p.poll() is not None:
+                break
+            if out:
+                logger.info(out)
+            if err:
+                logger.warning(err)
+            all_out += out
+            all_err += err
+    output = (all_out, all_err)
 
     if return_code:
         output = output + (p.returncode,)
@@ -158,17 +166,9 @@ def running_subprocess(command, after="wait", verbose=None, *args, **kwargs):
     """
     _validate_type(after, str, 'after')
     _check_option('after', after, ['wait', 'terminate', 'kill', 'communicate'])
-    for stdxxx, sys_stdxxx, thresh in (
-            ['stderr', sys.stderr, logging.ERROR],
-            ['stdout', sys.stdout, logging.WARNING]):
-        if stdxxx not in kwargs and logger.level >= thresh:
+    for stdxxx, sys_stdxxx in (['stderr', sys.stderr], ['stdout', sys.stdout]):
+        if stdxxx not in kwargs:
             kwargs[stdxxx] = subprocess.PIPE
-        elif kwargs.get(stdxxx, sys_stdxxx) is sys_stdxxx:
-            if isinstance(sys_stdxxx, StringIO):
-                # nose monkey patches sys.stderr and sys.stdout to StringIO
-                kwargs[stdxxx] = subprocess.PIPE
-            else:
-                kwargs[stdxxx] = sys_stdxxx
 
     # Check the PATH environment variable. If run_subprocess() is to be called
     # frequently this should be refactored so as to only check the path once.


Test patch: