Repository: xonsh/xonsh

Issue:
Sourcing virtualenvwrapper has no effect.
I use ROS and virtualenvwrapper, both of which need to be sourced from a sh file.

Here are the relevant parts of my `.xonshrc`

```
# ROS
source-bash /opt/ros/jade/setup.bash
source-bash ~/catkin_ws/devel/setup.bash

# Python Virtualenv
$WORKON_HOME = "~/.virtualenvs"
source-bash /usr/local/bin/virtualenvwrapper.sh
$PIP_VIRTUALENV_BASE = "~/.virtualenvs"
```

Sourcing ROS allows me to use `roscore`, for instance. However, what is expected for virtualenvwrapper is that, upon opening a new shell, I would be able to use commands such as `workon` or `mkvirtualenv`. Instead, I get `xonsh: subprocess mode: command not found` for all those commands.



Discussion:
Thanks for reporting, @masasin. Do the source-bash commands work when you type them into the interactive shell, rather than sticking them into the xonshrc? Just trying to figure out where the error is coming from

It does not. `$WORKON_HOME` and `$PIP_VIRTUALENV_BASE` register as normal, however.

Ahh Ok. I have figured this out.  So `workon` and `mkvirtualenv` are not commands, they are shell functions. We don't have great support for running external shell functions yet, but this is a pretty good use case. 


Resolving patch:
diff --git a/xonsh/aliases.py b/xonsh/aliases.py
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -9,10 +9,11 @@
 from xonsh.dirstack import cd, pushd, popd, dirs
 from xonsh.jobs import jobs, fg, bg, kill_all_jobs
 from xonsh.timings import timeit_alias
-from xonsh.tools import ON_MAC, ON_WINDOWS, XonshError
+from xonsh.tools import ON_MAC, ON_WINDOWS, XonshError, to_bool
 from xonsh.history import main as history_alias
 from xonsh.replay import main as replay_main
 from xonsh.environ import locate_binary
+from xonsh.foreign_shells import foreign_shell_data
 
 
 def exit(args, stdin=None):  # pylint:disable=redefined-builtin,W0622
@@ -23,32 +24,70 @@ def exit(args, stdin=None):  # pylint:disable=redefined-builtin,W0622
     return None, None
 
 
-def source_bash(args, stdin=None):
-    """Implements bash's source builtin."""
-    import tempfile
+_SOURCE_FOREIGN_PARSER = None
+
+def _ensure_source_foreign_parser():
+    global _SOURCE_FOREIGN_PARSER
+    if _SOURCE_FOREIGN_PARSER is not None:
+        return _SOURCE_FOREIGN_PARSER
+    desc = "Sources a file written in a foreign shell language."
+    parser = ArgumentParser('source-foreign', description=desc)
+    parser.add_argument('shell', help='Name or path to the foreign shell')
+    parser.add_argument('filenames', nargs='+', help='file paths to source')
+    parser.add_argument('-i', '--interactive', type=to_bool, default=True,
+                        help='whether the sourced shell should be interactive',
+                        dest='interactive')
+    parser.add_argument('-l', '--login', type=to_bool, default=False,
+                        help='whether the sourced shell should be login',
+                        dest='login')
+    parser.add_argument('--envcmd', default='env', dest='envcmd', 
+                        help='command to print environment')
+    parser.add_argument('--aliascmd', default='alias', dest='aliascmd', 
+                        help='command to print aliases')
+    parser.add_argument('--extra-args', default=(), dest='extra_args',
+                        type=(lambda s: tuple(s.split())), 
+                        help='extra arguments needed to run the shell')
+    parser.add_argument('-s', '--safe', type=to_bool, default=True, 
+                        help='whether the source shell should be run safely, '
+                             'and not raise any errors, even if they occur.',
+                        dest='safe')
+    parser.add_argument('--sourcer', default='source', dest='sourcer',
+                        help='the source command in the target shell language, '
+                             'default: source.')
+    _SOURCE_FOREIGN_PARSER = parser
+    return parser
+    
+
+def source_foreign(args, stdin=None):
+    """Sources a file written in a foreign shell language."""
+    parser = _ensure_source_foreign_parser()
+    ns = parser.parse_args(args)
+    prevcmd = '{0} {1}'.format(ns.sourcer, ' '.join(ns.filenames))
+    foreign_shell_data.cache_clear()  # make sure that we don't get prev src
+    fsenv, fsaliases = foreign_shell_data(shell=ns.shell, login=ns.login,
+                            interactive=ns.interactive, envcmd=ns.envcmd, 
+                            aliascmd=ns.aliascmd, extra_args=ns.extra_args,  
+                            safe=ns.safe, prevcmd=prevcmd)
+    # apply results
     env = builtins.__xonsh_env__
     denv = env.detype()
-    with tempfile.NamedTemporaryFile(mode='w+t') as f:
-        args = ' '.join(args)
-        inp = 'source {0}\nenv >> {1}\n'.format(args, f.name)
-        try:
-            subprocess.check_output(['bash'],
-                                    input=inp,
-                                    env=denv,
-                                    stderr=subprocess.PIPE,
-                                    universal_newlines=True)
-        except subprocess.CalledProcessError:
-            return None, 'could not source {0}\n'.format(args)
-        f.seek(0)
-        exported = f.read()
-    items = [l.split('=', 1) for l in exported.splitlines() if '=' in l]
-    newenv = dict(items)
-    for k, v in newenv.items():
+    for k, v in fsenv.items():
         if k in env and v == denv[k]:
             continue  # no change from original
         env[k] = v
-    return
+    baliases = builtins.aliases
+    for k, v in fsaliases.items():
+        if k in baliases and v == baliases[k]:
+            continue  # no change from original
+        baliases[k] = v
+
 
+def source_bash(args, stdin=None):
+    """Simple Bash-specific wrapper around source-foreign."""
+    args = list(args)
+    args.insert(0, 'bash')
+    args.append('--sourcer=source')
+    return source_foreign(args, stdin=stdin)
 
 def source_alias(args, stdin=None):
     """Executes the contents of the provided files in the current context.
@@ -118,6 +157,7 @@ def bang_bang(args, stdin=None):
     'xexec': xexec,
     'source': source_alias,
     'source-bash': source_bash,
+    'source-foreign': source_foreign,
     'history': history_alias,
     'replay': replay_main,
     '!!': bang_bang,
diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -13,20 +13,66 @@
 
 
 COMMAND = """
+{prevcmd}
 echo __XONSH_ENV_BEG__
 {envcmd}
 echo __XONSH_ENV_END__
 echo __XONSH_ALIAS_BEG__
 {aliascmd}
 echo __XONSH_ALIAS_END__
+echo __XONSH_FUNCS_BEG__
+{funcscmd}
+echo __XONSH_FUNCS_END__
+{postcmd}
 """.strip()
 
+DEFAULT_BASH_FUNCSCMD = """
+# get function names from declare
+declstr=$(declare -F)
+read -r -a decls <<< $declstr
+funcnames=""
+for((n=0;n<${#decls[@]};n++)); do
+  if (( $(($n % 3 )) == 2 )); then
+    # get every 3rd entry
+    funcnames="$funcnames ${decls[$n]}"
+  fi
+done
+
+# get functions locations: funcname lineno filename
+shopt -s extdebug
+namelocfilestr=$(declare -F $funcnames)
+shopt -u extdebug
+
+# print just name and file
+read -r -a namelocfile <<< $namelocfilestr
+namefile=""
+for((n=0;n<${#namelocfile[@]};n++)); do
+  if (( $(($n % 3 )) == 0 )); then
+    namefile="$namefile ${namelocfile[$n]}"
+  elif (( $(($n % 3 )) == 2 )); then
+    namefile="$namefile ${namelocfile[$n]}"
+  fi
+done
+echo $namefile
+""".strip()
+
+DEFAULT_FUNCSCMDS = {
+    'bash': DEFAULT_BASH_FUNCSCMD,
+    '/bin/bash': DEFAULT_BASH_FUNCSCMD,
+}
+
+DEFAULT_SOURCERS = {
+    'bash': 'source',
+    '/bin/bash': 'source',
+}
+
 @lru_cache()
 def foreign_shell_data(shell, interactive=True, login=False, envcmd='env', 
                        aliascmd='alias', extra_args=(), currenv=None, 
-                       safe=True):
+                       safe=True, prevcmd='', postcmd='', funcscmd=None,
+                       sourcer=None):
     """Extracts data from a foreign (non-xonsh) shells. Currently this gets 
-    the environment and aliases, but may be extended in the future.
+    the environment, aliases, and functions but may be extended in the future.
 
     Parameters
     ----------
@@ -45,14 +91,33 @@ def foreign_shell_data(shell, interactive=True, login=False, envcmd='env',
     currenv : tuple of items or None, optional
         Manual override for the current environment.
     safe : bool, optional
-        Flag for whether or not to safely handle exceptions and other errors. 
+        Flag for whether or not to safely handle exceptions and other errors.
+    prevcmd : str, optional
+        A command to run in the shell before anything else, useful for 
+        sourcing and other commands that may require environment recovery.
+    postcmd : str, optional
+        A command to run after everything else, useful for cleaning up any
+        damage that the prevcmd may have caused.
+    funcscmd : str or None, optional
+        This is a command or script that can be used to determine the names
+        and locations of any functions that are native to the foreign shell.
+        This command should print *only* a whitespace separated sequence
+        of pairs function name & filenames where the functions are defined.
+        If this is None, then a default script will attempted to be looked 
+        up based on the shell name. Callable wrappers for these functions
+        will be returned in the aliases dictionary.
+    sourcer : str or None, optional
+        How to source a foreign shell file for purposes of calling functions
+        in that shell. If this is None, a default value will attempt to be 
+        looked up based on the shell name.
 
     Returns
     -------
     env : dict
         Dictionary of shell's environment
     aliases : dict
-        Dictionary of shell's alaiases.
+        Dictionary of shell's alaiases, this includes foreign function 
+        wrappers.
     """
     cmd = [shell]
     cmd.extend(extra_args)  # needs to come here for GNU long options
@@ -61,13 +126,16 @@ def foreign_shell_data(shell, interactive=True, login=False, envcmd='env',
     if login:
         cmd.append('-l')
     cmd.append('-c')
-    cmd.append(COMMAND.format(envcmd=envcmd, aliascmd=aliascmd))
+    funcscmd = DEFAULT_FUNCSCMDS.get(shell, '') if funcscmd is None else funcscmd
+    command = COMMAND.format(envcmd=envcmd, aliascmd=aliascmd, prevcmd=prevcmd,
+                             postcmd=postcmd, funcscmd=funcscmd).strip()
+    cmd.append(command)
     if currenv is None and hasattr(builtins, '__xonsh_env__'):
         currenv = builtins.__xonsh_env__.detype()
     elif currenv is not None:
         currenv = dict(currenv)
     try:
-        s = subprocess.check_output(cmd,stderr=subprocess.PIPE, env=currenv,
+        s = subprocess.check_output(cmd, stderr=subprocess.PIPE, env=currenv,
                                     universal_newlines=True)
     except (subprocess.CalledProcessError, FileNotFoundError):
         if not safe:
@@ -75,6 +143,8 @@ def foreign_shell_data(shell, interactive=True, login=False, envcmd='env',
         return {}, {}
     env = parse_env(s)
     aliases = parse_aliases(s)
+    funcs = parse_funcs(s, shell=shell, sourcer=sourcer)
+    aliases.update(funcs)
     return env, aliases
 
 
@@ -120,8 +190,97 @@ def parse_aliases(s):
     return aliases
 
 
+FUNCS_RE = re.compile('__XONSH_FUNCS_BEG__\n(.*)__XONSH_FUNCS_END__',
+                      flags=re.DOTALL)
+
+def parse_funcs(s, shell, sourcer=None):
+    """Parses the funcs portion of a string into a dict of callable foreign
+    function wrappers.
+    """
+    m = FUNCS_RE.search(s)
+    if m is None:
+        return {}
+    g1 = m.group(1)
+    funcs = {}
+    flatpairs = g1.strip().split()
+    if len(flatpairs) % 2 != 0:
+        warn('could not parse functions, malformed pairs', RuntimeWarning)
+        return funcs
+    sourcer = DEFAULT_SOURCERS.get(shell, 'source') if sourcer is None \
+                                                    else sourcer
+    for funcname, filename in zip(flatpairs[::2], flatpairs[1::2]):
+        if funcname.startswith('_'):
+            continue  # skip private functions
+        wrapper = ForeignShellFunctionAlias(name=funcname, shell=shell, 
+                                            sourcer=sourcer, filename=filename)
+        funcs[funcname] = wrapper
+    return funcs
+
+
+class ForeignShellFunctionAlias(object):
+    """This class is responsible for calling foreign shell functions as if
+    they were aliases. This does not currently support taking stdin.
+    """
+
+    INPUT = ('{sourcer} {filename}\n'
+             '{funcname} {args}\n') 
+
+    def __init__(self, name, shell, filename, sourcer=None):
+        """
+        Parameters
+        ----------
+        name : str
+            function name
+        shell : str
+            Name or path to shell
+        filename : str
+            Where the function is defined, path to source.
+        sourcer : str or None, optional
+            Command to source foreing files with.
+        """
+        sourcer = DEFAULT_SOURCERS.get(shell, 'source') if sourcer is None \
+                                                        else sourcer
+        self.name = name
+        self.shell = shell
+        self.filename = filename
+        self.sourcer = sourcer
+
+    def __eq__(self, other):
+        if not hasattr(other, 'name') or not hasattr(other, 'shell') or \
+           not hasattr(other, 'filename') or not hasattr(other, 'sourcer'):
+            return NotImplemented
+        return (self.name == other.name) and (self.shell == other.shell) and \
+               (self.filename == other.filename) and (self.sourcer == other.sourcer)
+
+    def __call__(self, args, stdin=None):
+        args, streaming = self._is_streaming(args)
+        input = self.INPUT.format(sourcer=self.sourcer, filename=self.filename,
+                                  funcname=self.name, args=' '.join(args))
+        cmd = [self.shell, '-c', input]
+        env = builtins.__xonsh_env__
+        denv = env.detype()
+        if streaming:
+            subprocess.check_call(cmd, env=denv)
+            out = None
+        else:
+            out = subprocess.check_output(cmd, env=denv, stderr=subprocess.STDOUT)
+            out = out.decode(encoding=env.get('XONSH_ENCODING'),
+                             errors=env.get('XONSH_ENCODING_ERRORS'))
+            out = out.replace('\r\n', '\n')
+        return out
+
+    def _is_streaming(self, args):
+        """Test and modify args if --xonsh-stream is present."""
+        if '--xonsh-stream' not in args:
+            return args, False
+        args = list(args)
+        args.remove('--xonsh-stream')
+        return args, True
+
+
 VALID_SHELL_PARAMS = frozenset(['shell', 'interactive', 'login', 'envcmd', 
-                                'aliascmd', 'extra_args', 'currenv', 'safe'])
+                                'aliascmd', 'extra_args', 'currenv', 'safe', 
+                                'prevcmd', 'postcmd', 'funcscmd', 'sourcer'])
 
 def ensure_shell(shell):
     """Ensures that a mapping follows the shell specification."""
@@ -153,6 +312,16 @@ def ensure_shell(shell):
         shell['currenv'] = ce
     if 'safe' in shell_keys:
         shell['safe'] = to_bool(shell['safe'])
+    if 'prevcmd' in shell_keys:
+        shell['prevcmd'] = eunsure_string(shell['prevcmd'])
+    if 'postcmd' in shell_keys:
+        shell['postcmd'] = eunsure_string(shell['postcmd'])
+    if 'funcscmd' in shell_keys:
+        shell['funcscmd'] = None if shell['funcscmd'] is None \
+                                 else eunsure_string(shell['funcscmd'])
+    if 'sourcer' in shell_keys:
+        shell['sourcer'] = None if shell['sourcer'] is None \
+                                 else eunsure_string(shell['sourcer'])
     return shell
 
 


Test patch: