Repository: xonsh/xonsh

Issue:
Provide command to register unthreadable commands
See #2102

## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**

NixOS: coreutils are not captured because unthredable
_Originally posted by @amacfie in https://github.com/xonsh/xonsh/issues/4214#issuecomment-1328288341_:

On Debian/Arch:
```python
docker run --rm -it python:3.10-slim /bin/bash -c "pip install 'xonsh[full]' && xonsh"

xonsh --version  
# xonsh/0.13.3

!(echo hi) == 'hi\n'
# True
!(echo hi).out == 'hi\n'
# True
```
But on NixOS:
```bash
docker run --rm -it nixos/nix:2.3.16 /bin/sh

nix-channel --update
nix-shell -p xonsh

xonsh --version  
# xonsh/0.13.3
xonsh

!(echo hi) == 'hi\n'
# hi
# False
!(echo hi).out == 'hi\n'
# hi
# False
```
## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**
      


Discussion:

Might be that something in https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/programs/xonsh.nix (maybe in everything that gets `source`d) is screwing up our piping.
> Might be that something in https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/programs/xonsh.nix (maybe in everything that gets `source`d) is screwing up our piping.

Actually, the xonshrc defined in this module is not loaded by default without https://github.com/NixOS/nixpkgs/pull/204252, which was merged a few hours ago and is not yet in nixos-unstable (though will be landed very soon). I can reproduce the same behavior no matter whether /etc/xonshrc is loaded, so I don't think this issue is related to the NixOS module.
Thanks @SamLukeYes -- that at least rules out a bunch of things that would be very annoying to check.

The only thing left that I can think of is that we're getting some behavior from `subprocess.PIPE` that is different than what we expect.
This issue is specific to the `echo` command. On NixOS, when I run something like
```
!(python -c "print('hi')") == 'hi\n'
```
I get the expected output.

I think the cause of this issue is that xonsh thinks `echo` is not threadable, so it runs it using the `Popen` class instead of the `PopenThread` class and doesn't pipe its stdout/stderr:
https://github.com/xonsh/xonsh/blob/bcf2592b7dc628ba4f64bb3e9744199edc42cfa5/xonsh/procs/specs.py#L747-L761

I can also reproduce this issue on Fedora by running
```
!(cat /etc/passwd)
```
The `cat` command is not threadable, so its output is not captured.

Resolving patch:
diff --git a/xonsh/aliases.py b/xonsh/aliases.py
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -32,6 +32,7 @@
     ON_OPENBSD,
     ON_WINDOWS,
 )
+from xonsh.procs.specs import SpecAttrModifierAlias, SpecModifierAlias
 from xonsh.timings import timeit_alias
 from xonsh.tools import (
     ALIAS_KWARG_NAMES,
@@ -60,7 +61,7 @@ class FuncAlias:
     attributes_show = ["__xonsh_threadable__", "__xonsh_capturable__"]
     attributes_inherit = attributes_show + ["__doc__"]
 
-    def __init__(self, name, func):
+    def __init__(self, name, func=None):
         self.__name__ = self.name = name
         self.func = func
         for attr in self.attributes_inherit:
@@ -130,7 +131,7 @@ def wrapper(func):
 
         return wrapper
 
-    def get(self, key, default=None):
+    def get(self, key, default=None, spec_modifiers=None):
         """Returns the (possibly modified) value. If the key is not present,
         then `default` is returned.
         If the value is callable, it is returned without modification. If it
@@ -138,16 +139,21 @@ def get(self, key, default=None):
         other aliases, resulting in a new list or a "partially applied"
         callable.
         """
+        spec_modifiers = spec_modifiers if spec_modifiers is not None else []
         val = self._raw.get(key)
         if val is None:
             return default
         elif isinstance(val, cabc.Iterable) or callable(val):
-            return self.eval_alias(val, seen_tokens={key})
+            return self.eval_alias(
+                val, seen_tokens={key}, spec_modifiers=spec_modifiers
+            )
         else:
             msg = "alias of {!r} has an inappropriate type: {!r}"
             raise TypeError(msg.format(key, val))
 
-    def eval_alias(self, value, seen_tokens=frozenset(), acc_args=()):
+    def eval_alias(
+        self, value, seen_tokens=frozenset(), acc_args=(), spec_modifiers=None
+    ):
         """
         "Evaluates" the alias ``value``, by recursively looking up the leftmost
         token and "expanding" if it's also an alias.
@@ -158,8 +164,17 @@ def eval_alias(self, value, seen_tokens=frozenset(), acc_args=()):
         callable.  The resulting callable will be "partially applied" with
         ``["-al", "arg"]``.
         """
+        spec_modifiers = spec_modifiers if spec_modifiers is not None else []
         # Beware of mutability: default values for keyword args are evaluated
         # only once.
+        if (
+            isinstance(value, cabc.Iterable)
+            and len(value) > 1
+            and (isinstance(mod := self._raw.get(str(value[0])), SpecModifierAlias))
+        ):
+            spec_modifiers.append(mod)
+            value = value[1:]
+
         if callable(value):
             return partial_eval_alias(value, acc_args=acc_args)
         else:
@@ -176,7 +191,12 @@ def eval_alias(self, value, seen_tokens=frozenset(), acc_args=()):
             else:
                 seen_tokens = seen_tokens | {token}
                 acc_args = rest + list(acc_args)
-                return self.eval_alias(self._raw[token], seen_tokens, acc_args)
+                return self.eval_alias(
+                    self._raw[token],
+                    seen_tokens,
+                    acc_args,
+                    spec_modifiers=spec_modifiers,
+                )
 
     def expand_alias(self, line: str, cursor_index: int) -> str:
         """Expands any aliases present in line if alias does not point to a
@@ -918,6 +938,14 @@ def make_default_aliases():
         "completer": xca.completer_alias,
         "xpip": detect_xpip_alias(),
         "xonsh-reset": xonsh_reset,
+        "xthread": SpecAttrModifierAlias(
+            {"threadable": True, "force_threadable": True},
+            "Mark current command as threadable.",
+        ),
+        "xunthread": SpecAttrModifierAlias(
+            {"threadable": False, "force_threadable": False},
+            "Mark current command as unthreadable.",
+        ),
     }
     if ON_WINDOWS:
         # Borrow builtin commands from cmd.exe.
diff --git a/xonsh/procs/pipelines.py b/xonsh/procs/pipelines.py
--- a/xonsh/procs/pipelines.py
+++ b/xonsh/procs/pipelines.py
@@ -165,7 +165,9 @@ def __init__(self, specs):
             # taking the terminal away from the `less` command, causing `less`
             # to stop.
             pipeline_group = os.getpgid(0)
-        for spec in specs:
+        for i, spec in enumerate(specs):
+            for mod in spec.spec_modifiers:
+                mod.on_pre_run(self, spec, i)
             if self.starttime is None:
                 self.starttime = time.time()
             try:
diff --git a/xonsh/procs/specs.py b/xonsh/procs/specs.py
--- a/xonsh/procs/specs.py
+++ b/xonsh/procs/specs.py
@@ -277,6 +277,45 @@ def no_pg_xonsh_preexec_fn():
     signal.signal(signal.SIGTSTP, default_signal_pauser)
 
 
+class SpecModifierAlias:
+    """Spec modifier base class."""
+
+    descr = "Spec modifier base class."
+
+    def __call__(
+        self,
+        args,
+        stdin=None,
+        stdout=None,
+        stderr=None,
+        spec=None,
+        stack=None,
+        **kwargs,
+    ):
+        print(self.descr, file=stdout)
+
+    def on_modifer_added(self, spec):
+        """Modify spec immediately after modifier added."""
+        pass
+
+    def on_pre_run(self, pipeline, spec, spec_num):
+        """Modify spec before run."""
+        pass
+
+
+class SpecAttrModifierAlias(SpecModifierAlias):
+    """Modifier for spec attributes."""
+
+    def __init__(self, set_attributes: dict, descr=""):
+        self.set_attributes = set_attributes
+        self.descr = descr
+        super().__init__()
+
+    def on_modifer_added(self, spec):
+        for a, v in self.set_attributes.items():
+            setattr(spec, a, v)
+
+
 class SubprocSpec:
     """A container for specifying how a subprocess command should be
     executed.
@@ -380,6 +419,7 @@ def __init__(
         self.captured_stdout = None
         self.captured_stderr = None
         self.stack = None
+        self.spec_modifiers = []  # List of SpecModifierAlias objects that applied to spec.
 
     def __str__(self):
         s = self.__class__.__name__ + "(" + str(self.cmd) + ", "
@@ -600,6 +640,7 @@ def build(kls, cmd, *, cls=subprocess.Popen, **kwargs):
         # modifications that do not alter cmds may come before creating instance
         spec = kls(cmd, cls=cls, **kwargs)
         # modifications that alter cmds must come after creating instance
+        spec.resolve_spec_modifiers()  # keep this first
         spec.resolve_args_list()
         spec.resolve_redirects()
         spec.resolve_alias()
@@ -610,6 +651,25 @@ def build(kls, cmd, *, cls=subprocess.Popen, **kwargs):
         spec.resolve_stack()
         return spec
 
+    def add_spec_modifier(self, mod: SpecModifierAlias):
+        """Add spec modifier to the specification."""
+        mod.on_modifer_added(self)
+        self.spec_modifiers.append(mod)
+
+    def resolve_spec_modifiers(self):
+        """Apply spec modifier."""
+        if (ln := len(self.cmd)) == 1:
+            return
+        for i in range(ln):
+            c = self.cmd[i]
+            if c in XSH.aliases and isinstance(
+                mod := XSH.aliases[c], SpecModifierAlias
+            ):
+                self.add_spec_modifier(mod)
+            else:
+                break
+        self.cmd = self.cmd[i:]
+
     def resolve_args_list(self):
         """Weave a list of arguments into a command."""
         resolved_cmd = []
@@ -655,9 +715,17 @@ def resolve_alias(self):
         if callable(cmd0):
             alias = cmd0
         else:
-            alias = XSH.aliases.get(cmd0, None)
+            spec_modifiers = []
+            if isinstance(XSH.aliases, dict):
+                # Windows tests
+                alias = XSH.aliases.get(cmd0, None)
+            else:
+                alias = XSH.aliases.get(cmd0, None, spec_modifiers=spec_modifiers)
             if alias is not None:
                 self.alias_name = cmd0
+            if spec_modifiers:
+                for mod in spec_modifiers:
+                    self.add_spec_modifier(mod)
         self.alias = alias
 
     def resolve_binary_loc(self):
@@ -729,7 +797,8 @@ def resolve_stack(self):
         # we want to filter out one up, e.g. subproc_captured_hiddenobject()
         # after that the stack from the call site starts.
         stack = inspect.stack(context=0)
-        assert stack[3][3] == "run_subproc", "xonsh stack has changed!"
+        if not stack[3][3].startswith("test_"):
+            assert stack[3][3] == "run_subproc", "xonsh stack has changed!"
         del stack[:5]
         self.stack = stack
 


Test patch:
diff --git a/tests/procs/test_specs.py b/tests/procs/test_specs.py
--- a/tests/procs/test_specs.py
+++ b/tests/procs/test_specs.py
@@ -10,6 +10,7 @@
 from xonsh.procs.posix import PopenThread
 from xonsh.procs.proxies import STDOUT_DISPATCHER, ProcProxy, ProcProxyThread
 from xonsh.procs.specs import (
+    SpecAttrModifierAlias,
     SubprocSpec,
     _run_command_pipeline,
     cmds_to_specs,
@@ -169,6 +170,7 @@ def test_interrupted_process_returncode(xonsh_session, captured, interactive):
         [["echo", "1"], "|", cmd_sig("SIGTTIN"), "|", ["head"]],
     ],
 )
+@pytest.mark.flaky(reruns=3, reruns_delay=1)
 def test_specs_with_suspended_captured_process_pipeline(
     xonsh_session, suspended_pipeline
 ):
@@ -219,6 +221,55 @@ def test_run_subproc_background(captured, exp_is_none):
     assert (return_val is None) == exp_is_none
 
 
+def test_spec_modifier_alias_alone(xession):
+    xession.aliases["xunthread"] = SpecAttrModifierAlias(
+        {"threadable": False, "force_threadable": False}
+    )
+
+    cmds = [["xunthread"]]
+    spec = cmds_to_specs(cmds, captured="object")[-1]
+
+    assert spec.cmd == []
+    assert spec.alias_name == "xunthread"
+
+
+def test_spec_modifier_alias(xession):
+    xession.aliases["xunthread"] = SpecAttrModifierAlias(
+        {"threadable": False, "force_threadable": False}
+    )
+
+    cmds = [["xunthread", "echo", "arg0", "arg1"]]
+    spec = cmds_to_specs(cmds, captured="object")[-1]
+
+    assert spec.cmd == ["echo", "arg0", "arg1"]
+    assert spec.threadable is False
+    assert spec.force_threadable is False
+
+
+def test_spec_modifier_alias_tree(xession):
+    xession.aliases["xthread"] = SpecAttrModifierAlias(
+        {"threadable": True, "force_threadable": True}
+    )
+    xession.aliases["xunthread"] = SpecAttrModifierAlias(
+        {"threadable": False, "force_threadable": False}
+    )
+
+    xession.aliases["foreground"] = "xthread midground f0 f1"
+    xession.aliases["midground"] = "ground m0 m1"
+    xession.aliases["ground"] = "xthread underground g0 g1"
+    xession.aliases["underground"] = "xunthread echo u0 u1"
+
+    cmds = [
+        ["foreground"],
+    ]
+    spec = cmds_to_specs(cmds, captured="object")[-1]
+
+    assert spec.cmd == ["echo", "u0", "u1", "g0", "g1", "m0", "m1", "f0", "f1"]
+    assert spec.alias_name == "foreground"
+    assert spec.threadable is False
+    assert spec.force_threadable is False
+
+
 @pytest.mark.parametrize("thread_subprocs", [False, True])
 def test_callable_alias_cls(thread_subprocs, xession):
     class Cls: