Repository: oils-for-unix/oils
Pull request #2292: io.captureAll() Method (return stdout, stderr and status)
Author: login: Melkor333
Created: 2025-03-21T18:45:04Z | Merged: 2025-04-21T14:42:54Z



Diff:
diff --git a/builtin/method_io.py b/builtin/method_io.py
index 80a5a61337..55f2fac2ca 100644
--- a/builtin/method_io.py
+++ b/builtin/method_io.py
@@ -232,6 +232,29 @@ def Call(self, rd):
         return value.Str(stdout_str)
 
 
+class CaptureAll(vm._Callable):
+    def __init__(self, mem, shell_ex):
+        # type: (state.Mem, vm._Executor) -> None
+        self.mem = mem
+        self.shell_ex = shell_ex
+
+    def Call(self, rd):
+        # type: (typed_args.Reader) -> value_t
+        unused = rd.PosValue()
+        cmd = rd.PosCommand()
+        rd.Done()  # no more args
+        frag = typed_args.GetCommandFrag(cmd)
+        with state.ctx_EnclosedFrame(self.mem, cmd.captured_frame,
+                                     cmd.module_frame, None):
+            status, stdout_str, stderr_str = self.shell_ex.Capture3(frag)
+
+        out = NewDict() # type: Dict[str, value_t]
+        out['stdout'] = value.Str(stdout_str)
+        out['stderr'] = value.Str(stderr_str)
+        out['status'] = num.ToBig(status)
+
+        return value.Dict(out)
+
 class PromptVal(vm._Callable):
     """
     _io->promptVal('$') is like \$ 
diff --git a/core/executor.py b/core/executor.py
index e4f7a0f9db..4af899d26b 100644
--- a/core/executor.py
+++ b/core/executor.py
@@ -272,6 +272,13 @@ def CaptureStdout(self, node):
         """
         return 0, ''
 
+    def Capture3(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+        """
+        Used by io->captureAll() method, and called by command sub
+        """
+        return 0, '', ''
+
     def RunCommandSub(self, cs_part):
         # type: (CommandSub) -> str
         raise error.Structured(
@@ -637,6 +644,62 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def Capture3(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        open_fds = [stdout_fd, stderr_fd]
+        while True:
+            fds = pyos.WaitForReading(open_fds)
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    open_fds.remove(fd)
+
+            if len(open_fds) == 0:
+                break
+
+        posix.close(stdout_fd)
+        posix.close(stderr_fd)
+
+        status = p.Wait(self.waiter)
+        stdout_str = ''.join(stdout_chunks)
+        stderr_str = ''.join(stderr_chunks)
+
+        return status, stdout_str, stderr_str
+
     def RunCommandSub(self, cs_part):
         # type: (CommandSub) -> str
 
diff --git a/core/process.py b/core/process.py
index b3b92a9f11..c3f3722ff5 100644
--- a/core/process.py
+++ b/core/process.py
@@ -629,6 +629,26 @@ def Apply(self):
         #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
 
 
+class StderrToPipe(ChildStateChange):
+
+    def __init__(self, r, pipe_write_fd):
+        # type: (int, int) -> None
+        self.r = r
+        self.w = pipe_write_fd
+
+    def __repr__(self):
+        # type: () -> str
+        return '<StderrToPipe %d %d>' % (self.r, self.w)
+
+    def Apply(self):
+        # type: () -> None
+        posix.dup2(self.w, 2)
+        posix.close(self.w)  # close after dup
+
+        posix.close(self.r)  # we're writing to the pipe, not reading
+        #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
+
+
 INVALID_PGID = -1
 # argument to setpgid() that means the process is its own leader
 OWN_LEADER = 0
diff --git a/core/pyos.py b/core/pyos.py
index 23312f5299..28f8a21feb 100644
--- a/core/pyos.py
+++ b/core/pyos.py
@@ -276,12 +276,18 @@ def OsType():
 
 def InputAvailable(fd):
     # type: (int) -> bool
-    # similar to lib/sh/input_avail.c in bash
     # read, write, except
-    r, w, exc = select.select([fd], [], [fd], 0)
+    r, _, _ = select.select([fd], [], [], 0)
     return len(r) != 0
 
 
+def WaitForReading(fd_list):
+    # type: (List[int]) -> List[int]
+    # read, write, except
+    r, _, _ = select.select(fd_list, [], [])
+    return r
+
+
 def MakeDirCacheKey(path):
     # type: (str) -> Tuple[str, int]
     """Returns a pair (path with last modified time) that can be used to cache
diff --git a/core/shell.py b/core/shell.py
index 793f45d406..c8fdbf026d 100644
--- a/core/shell.py
+++ b/core/shell.py
@@ -536,6 +536,9 @@ def Main(
     # Identical to command sub
     io_methods['captureStdout'] = value.BuiltinFunc(
         method_io.CaptureStdout(mem, shell_ex))
+    # Like captureStdout but capture stderr, too
+    io_methods['captureAll'] = value.BuiltinFunc(
+        method_io.CaptureAll(mem, shell_ex))
 
     # TODO: remove these 2 deprecated methods
     io_methods['M/evalToDict'] = value.BuiltinFunc(
diff --git a/core/vm.py b/core/vm.py
index 39ad99bc77..0094a2f7a9 100644
--- a/core/vm.py
+++ b/core/vm.py
@@ -332,6 +332,10 @@ def CaptureStdout(self, node):
         # type: (command_t) -> Tuple[int, str]
         return 0, ''
 
+    def Capture3(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+        return 0, '', ''
+
     def RunCommandSub(self, cs_part):
         # type: (CommandSub) -> str
         return ''
diff --git a/cpp/core.cc b/cpp/core.cc
index fb8bb207f3..9acc1c706f 100644
--- a/cpp/core.cc
+++ b/cpp/core.cc
@@ -254,6 +254,27 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForReading(List<int>* fd_list) {
+  auto* ret = NewList<int>();
+  fd_set fd_bits;
+  FD_ZERO(&fd_bits);
+  int n = len(fd_list);
+  for (int i = 0; i < n; ++i) {
+    FD_SET(fd_list->at(i), &fd_bits);
+  }
+  int num_ready = select(FD_SETSIZE, &fd_bits, NULL, NULL, NULL);
+  for (int i = 0; i < n; ++i) {
+    if (FD_ISSET(fd_list->at(i), &fd_bits)) {
+      ret->append(fd_list->at(i));
+    }
+    if (len(ret) == num_ready) {
+      break;
+    }
+  }
+  return ret;
+}
+
+
 IOError_OSError* FlushStdout() {
   // Flush libc buffers
   if (::fflush(stdout) != 0) {
diff --git a/cpp/core.h b/cpp/core.h
index f6d106b256..873964363d 100644
--- a/cpp/core.h
+++ b/cpp/core.h
@@ -80,6 +80,8 @@ void PrintTimes();
 
 bool InputAvailable(int fd);
 
+List<int>* WaitForReading(List<int>* fd_list);
+
 IOError_OSError* FlushStdout();
 
 Tuple2<int, void*> PushTermAttrs(int fd, int mask);
diff --git a/demo/slow.ysh b/demo/slow.ysh
new file mode 100755
index 0000000000..8e2bedaedf
--- /dev/null
+++ b/demo/slow.ysh
@@ -0,0 +1,19 @@
+#!bin/ysh
+
+source $LIB_YSH/list.ysh
+
+var captured = 'captured'
+
+proc p (;n) {
+  for i in (0 ..< n) {
+    write -- $[join(repeat([i], i))]
+    echo x >&2
+    sleep 0.01
+  }
+}
+
+var cmd = ^( p (50) )
+var result = io.captureAll(cmd)
+= result
+
+assert [repeat(b'x\n', 50) === result.stderr]
diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md
index edc7e230eb..6967ad13ef 100644
--- a/doc/ref/chap-type-method.md
+++ b/doc/ref/chap-type-method.md
@@ -951,10 +951,10 @@ To evaluate "purely", use the [`evalExpr()`][func/evalExpr] function.
 
 ### captureStdout()
 
-Capture stdout of a command a string.
+Run a Command, and return its stdout as a astring.
 
     var c = ^(echo hi)
-    var stdout_str = _io.captureStdout(c)  # => "hi"
+    var stdout_str = io.captureStdout(c)  # => "hi"
 
 It's like `$()` [command subs][command-sub], but can be used in pure functions
 that have access to `io`.
@@ -967,7 +967,28 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
+    }
+
+### captureAll()
+
+Run a Command, and return its `stdout` string, `stderr` string, and integer
+`status`.
+
+    var c = ^(echo out; echo err >&2)
+    var r = io.captureAll(c)  # => { stdout: "out", stderr: "err", status: 0 }
+
+It's similar to `io.captureStdout`, but returns more info.
+
+- NUL bytes and trailing newlines `\n` are **not** removed.
+- Because it captures the status, it doesn't fail when the status is non-zero.
+
+    = io.captureAll(^(echo stdout; echo stderr >&2; exit 3))
+    (Dict)
+    {
+        stdout: 'stdout\n',
+        stderr: 'stderr\n',
+        status: 3
     }
 
 ### promptVal()
diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md
index c349448e4d..c38109032c 100644
--- a/doc/ref/toc-ysh.md
+++ b/doc/ref/toc-ysh.md
@@ -70,7 +70,7 @@ error handling, and more.
                    Frame
                    DebugFrame  toString()
                    io          stdin            io/eval()         io/evalExpr()
-                               captureStdout()  promptVal()
+                               captureStdout()  captureAll()      promptVal()
                              X time()         X strftime()      X glob()
                    vm          getFrame()       getDebugStack()   id() 
 ```
diff --git a/shell.nix b/shell.nix
index 95749a0335..eb3218a764 100644
--- a/shell.nix
+++ b/shell.nix
@@ -64,7 +64,7 @@ let
 
   binary = [ re2c ];
   doctools = [ cmark ];
-  c_deps = [ readline ];
+  c_deps = [ readline70 ];
 
   shell_deps = [
     gawk
@@ -97,7 +97,8 @@ in mkShell rec {
   # weigh in if there's a better way to handle this.
   #
   # Note: Nix automatically adds identifiers declared here to the environment!
-  _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
+  _NIX_SHELL_LIBCMARK =
+    "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
 
   # Need nix to relax before it'll link against a local file.
   NIX_ENFORCE_PURITY = 0;
diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh
index a7008f68dd..eae7e553ab 100644
--- a/spec/ysh-blocks.test.sh
+++ b/spec/ysh-blocks.test.sh
@@ -1,4 +1,4 @@
-## oils_failures_allowed: 2
+## oils_failures_allowed: 1
 
 #### cd accepts a block, runs it in different dir
 shopt -s ysh:all
@@ -79,7 +79,7 @@ p
 ## STDOUT:
 ## END
 
-#### io->eval() and io.captureStdout() passed a block in different scope
+#### io->eval() and io.captureStdout()/io.captureAll() passed a block in different scope
 shopt --set ysh:upgrade
 
 proc my-cd (; b) {
@@ -92,6 +92,9 @@ proc my-cd (; b) {
   # Yup, this is a problem
   var s = io.captureStdout(b)
   echo "stdout $s"
+
+  setvar s = io.captureAll(b)
+  echo "stdout $[s.stdout]" # Adds a newline
 }
 
 proc p {
@@ -104,6 +107,12 @@ proc p {
 p
 
 ## STDOUT:
+i = 42
+i = 42
+(Dict)   {"x":"x"}
+stdout i = 42
+stdout i = 42
+
 ## END
 
 #### block doesn't have its own scope
diff --git a/spec/ysh-method-io.test.sh b/spec/ysh-method-io.test.sh
index 0df79d64aa..511af64317 100644
--- a/spec/ysh-method-io.test.sh
+++ b/spec/ysh-method-io.test.sh
@@ -51,6 +51,49 @@ two
 status=null
 ## END
 
+#### captureAll() is like $() but captures stderr and status
+
+proc p {
+  var captured = 'captured'
+  var cmd = ^(
+    echo stdout
+    echo stderr >&2
+    echo $captured
+    echo $captured >&2
+    printf '\000'
+    exit 42
+  )
+
+  var result = io.captureAll(cmd)
+  pp test_ (result)
+}
+
+p
+
+## STDOUT:
+(Dict)   {"stdout":"stdout\ncaptured\n\u0000","stderr":"stderr\ncaptured\n","status":42}
+## END
+
+#### captureAll() doesn't fail
+
+var c = ^(shopt --unset verbose_errexit; echo out; echo err >&2; false; echo two)
+
+# Hm this prints a message, but no stack trace
+# Should make it fail I think
+
+try {
+  var x = io.captureAll(c)
+}
+# This has {"code": 3} because it's an expression error.  Should probably
+pp test_ (_error)
+pp test_ (x)
+
+## status: 0
+## STDOUT:
+(Dict)   {"code":0}
+(Dict)   {"stdout":"out\n","stderr":"err\n","status":1}
+## END
+
 #### io->eval() with failing command - caller must handle
 
 var c = ^(echo one; false; echo two)


Reviews:
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T10:56:55Z
user: login: andychu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T03:53:47Z
user: login: andychu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T03:12:59Z
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T13:01:07Z
user: login: andychu
body: OK thanks, this looks good!  Just some naming issues mainly

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-30T16:46:31Z
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T19:56:49Z
user: login: andychu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T02:31:31Z
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T04:35:14Z
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T06:24:17Z
user: login: andychu
body: OK looks great!  Just some minor cosmetic things

Thank you

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T17:02:59Z
user: login: Melkor333
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-05T08:15:38Z
user: login: andychu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-05T17:39:47Z

Review comments:
diff_hunk: @@ -595,41 +596,59 @@ def RunSubshell(self, node):
         return p.RunProcess(self.waiter, trace.ForkWait)
 
     def CaptureStdout(self, node):
-        # type: (command_t) -> Tuple[int, str]
+        # type: (command_t) -> Tuple[int, str, str]
 
         p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
                               self.exec_opts.errtrace())
         # Shell quirk: Command subs remain part of the shell's process group, so we
         # don't use p.AddStateChange(process.SetPgid(...))
 
         r, w = posix.pipe()
+        r2, w2 = posix.pipe()
         p.AddStateChange(process.StdoutToPipe(r, w))
+        p.AddStateChange(process.StderrToPipe(r2, w2))
 
         p.StartProcess(trace.CommandSub)
         #log('Command sub started %d', pid)
 
         chunks = []  # type: List[str]
+        chunks2 = []  # type: List[str]
         posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
         while True:
-            n, err_num = pyos.Read(r, 4096, chunks)
+            fds, w, exc = select.select([r,r2], [], [])
path: core/executor.py
user: login: Melkor333
body: @andychu I get the following error and don't know why 😬 

```
core/executor.py:621: error: Incompatible types in assignment (expression has type "List[Any]", variable has type "int")
```

Do you have an idea why this doesn't work here? I don't really get where it gets an `int` when I pass 3 lists...

(And don't worry, I'll make it a deparate function from `captureStdout`. I just wanted to test if it works that way on the phone and wanna clean it up on my laptop once it works)
created_at: 2025-03-23T10:56:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -595,41 +596,59 @@ def RunSubshell(self, node):
         return p.RunProcess(self.waiter, trace.ForkWait)
 
     def CaptureStdout(self, node):
-        # type: (command_t) -> Tuple[int, str]
+        # type: (command_t) -> Tuple[int, str, str]
 
         p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
                               self.exec_opts.errtrace())
         # Shell quirk: Command subs remain part of the shell's process group, so we
         # don't use p.AddStateChange(process.SetPgid(...))
 
         r, w = posix.pipe()
+        r2, w2 = posix.pipe()
         p.AddStateChange(process.StdoutToPipe(r, w))
+        p.AddStateChange(process.StderrToPipe(r2, w2))
 
         p.StartProcess(trace.CommandSub)
         #log('Command sub started %d', pid)
 
         chunks = []  # type: List[str]
+        chunks2 = []  # type: List[str]
         posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
         while True:
-            n, err_num = pyos.Read(r, 4096, chunks)
+            fds, w, exc = select.select([r,r2], [], [])
path: core/executor.py
user: login: andychu
body: Oh sorry I didn't respond to this earlier ...

I think MyPy has problems inferring the type of the empty list `[]`.  So it just makes it  `List[Any]`

Sometimes I have to do this:

```
tmp = []  # type: List[int]
tmp2 = []  # type: List[int]
```


created_at: 2025-03-27T03:53:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -254,6 +254,26 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForInputs(List<int>* fds_in) {
+  auto* ret = NewList<int>();
+  fd_set fds;
+  FD_ZERO(&fds);
+  int n = len(fds_in);
+  for (int i = 0; i < n; ++i) {
+    FD_SET(fds_in->at(n), &fds);
+  }
+  int out = select(FD_SETSIZE, &fds, NULL, NULL, NULL);
+  for (int i = 0; i < n; ++i) {
+    if (InputAvailable(fds_in->at(n))) {
+      ret->append(fds_in->at(n));
path: cpp/core.cc
user: login: andychu
body: Hm I wonder if this is causing the IndexError

I don't think you should call `InputAvailable()` here ... the `select()` is supposed to return the list of descriptors

I think you make a new list from what select "returns", unconditionally

created_at: 2025-03-28T03:10:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -595,41 +595,62 @@ def RunSubshell(self, node):
         return p.RunProcess(self.waiter, trace.ForkWait)
 
     def CaptureStdout(self, node):
-        # type: (command_t) -> Tuple[int, str]
+        # type: (command_t) -> Tuple[int, str, str]
 
         p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
                               self.exec_opts.errtrace())
         # Shell quirk: Command subs remain part of the shell's process group, so we
         # don't use p.AddStateChange(process.SetPgid(...))
 
         r, w = posix.pipe()
+        r2, w2 = posix.pipe()
         p.AddStateChange(process.StdoutToPipe(r, w))
+        p.AddStateChange(process.StderrToPipe(r2, w2))
 
         p.StartProcess(trace.CommandSub)
         #log('Command sub started %d', pid)
 
         chunks = []  # type: List[str]
+        chunks2 = []  # type: List[str]
         posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
+        finished = False
         while True:
-            n, err_num = pyos.Read(r, 4096, chunks)
+            fds = pyos.WaitForInputs([r, r2])
 
-            if n < 0:
-                if err_num == EINTR:
-                    pass  # retry
-                else:
-                    # Like the top level IOError handler
-                    e_die_status(
-                        2,
-                        'Oils I/O error (read): %s' % posix.strerror(err_num))
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
 
-            elif n == 0:  # EOF
+            for fd in fds:
+                if fd == r:
path: core/executor.py
user: login: andychu
body: what is this condition for?

created_at: 2025-03-28T03:12:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -595,41 +595,62 @@ def RunSubshell(self, node):
         return p.RunProcess(self.waiter, trace.ForkWait)
 
     def CaptureStdout(self, node):
-        # type: (command_t) -> Tuple[int, str]
+        # type: (command_t) -> Tuple[int, str, str]
 
         p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
                               self.exec_opts.errtrace())
         # Shell quirk: Command subs remain part of the shell's process group, so we
         # don't use p.AddStateChange(process.SetPgid(...))
 
         r, w = posix.pipe()
+        r2, w2 = posix.pipe()
         p.AddStateChange(process.StdoutToPipe(r, w))
+        p.AddStateChange(process.StderrToPipe(r2, w2))
 
         p.StartProcess(trace.CommandSub)
         #log('Command sub started %d', pid)
 
         chunks = []  # type: List[str]
+        chunks2 = []  # type: List[str]
         posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
+        finished = False
         while True:
-            n, err_num = pyos.Read(r, 4096, chunks)
+            fds = pyos.WaitForInputs([r, r2])
 
-            if n < 0:
-                if err_num == EINTR:
-                    pass  # retry
-                else:
-                    # Like the top level IOError handler
-                    e_die_status(
-                        2,
-                        'Oils I/O error (read): %s' % posix.strerror(err_num))
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
 
-            elif n == 0:  # EOF
+            for fd in fds:
+                if fd == r:
path: core/executor.py
user: login: Melkor333
body: Made it more clear now. It checks if it's the `stdout_fd` and writes to the `stdout_chunks`, otherwise to the `stderr_chunks`.
created_at: 2025-03-28T13:01:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -232,6 +232,41 @@ def Call(self, rd):
         return value.Str(stdout_str)
 
 
+class CaptureOutputs(vm._Callable):
+    def __init__(self, mem, shell_ex):
+        # type: (state.Mem, vm._Executor) -> None
+        self.mem = mem
+        self.shell_ex = shell_ex
+
+    def Call(self, rd):
+        # type: (typed_args.Reader) -> value_t
+        unused = rd.PosValue()
+        cmd = rd.PosCommand()
+        # Allow us to not fail on error
+        fail = rd.NamedBool("fail", True)
path: builtin/method_io.py
user: login: andychu
body: I think it might be OK to NEVER fail, because returning the status sort of implies that

created_at: 2025-03-30T16:28:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -632,6 +639,64 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def CaptureOutputs(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
+        finished = False
+        while True:
+            fds = pyos.WaitForInputs([stdout_fd, stderr_fd])
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    eofcount += 1
+                if eofcount > 1:
path: core/executor.py
user: login: andychu
body: I would say `eofcount == 2`, and then `assert 0 <= eofcount < 2`

created_at: 2025-03-30T16:31:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -282,6 +282,14 @@ def InputAvailable(fd):
     return len(r) != 0
 
 
+def WaitForInputs(fd):
+    # type: (List[int]) -> List[int]
+    # Get a list of fds and return fds which have available input
+    # read, write, except
+    r, w, exc = select.select(fd, [], fd)
path: core/pyos.py
user: login: andychu
body: Hm I want to see what we should do about `exc`

Well at least we can see what other programs / shells do

from `man select`, I guess none of thse apply since it is a pipe ...  well that could be in a comment here



```
       POLLPRI
              There is some exceptional condition on the file descriptor.  Possibilities include:

              •  There is out-of-band data on a TCP socket (see tcp(7)).

              •  A pseudoterminal master in packet mode has seen a state change on the slave (see ioctl_tty(2)).

              •  A cgroup.events file has been modified (see cgroups(7)).
```
created_at: 2025-03-30T16:34:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -282,6 +282,14 @@ def InputAvailable(fd):
     return len(r) != 0
 
 
+def WaitForInputs(fd):
+    # type: (List[int]) -> List[int]
+    # Get a list of fds and return fds which have available input
+    # read, write, except
+    r, w, exc = select.select(fd, [], fd)
path: core/pyos.py
user: login: andychu
body: If we're not going to use `w` and `exc`, then we can call them `_, _`

Also, if we ignore `exc`, then I guess we should not pass the third param to select



created_at: 2025-03-30T16:36:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -282,6 +282,14 @@ def InputAvailable(fd):
     return len(r) != 0
 
 
+def WaitForInputs(fd):
path: core/pyos.py
user: login: andychu
body: fd implie a single descriptor -- I would call this `fd_list`

Some other names could be `WaitToRead(fd_list)` or `WaitForReading(fd_list)`

created_at: 2025-03-30T16:37:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -254,6 +254,27 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForInputs(List<int>* fds_in) {
+  auto* ret = NewList<int>();
+  fd_set fds;
+  FD_ZERO(&fds);
+  int n = len(fds_in);
+  for (int i = 0; i < n; ++i) {
+    FD_SET(fds_in->at(i), &fds);
+  }
+  int out = select(FD_SETSIZE, &fds, NULL, NULL, NULL);
path: cpp/core.cc
user: login: andychu
body: yeah here we don't pass the list of "exc"?   So we probably shouldn't pass it in Python, if there are no errors to handle

I'm not sure if that affect "close" or not ...

created_at: 2025-03-30T16:38:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -254,6 +254,27 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForInputs(List<int>* fds_in) {
+  auto* ret = NewList<int>();
+  fd_set fds;
+  FD_ZERO(&fds);
+  int n = len(fds_in);
+  for (int i = 0; i < n; ++i) {
+    FD_SET(fds_in->at(i), &fds);
+  }
+  int out = select(FD_SETSIZE, &fds, NULL, NULL, NULL);
path: cpp/core.cc
user: login: andychu
body: `out` should be something like `num_ready`

created_at: 2025-03-30T16:40:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -254,6 +254,27 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForInputs(List<int>* fds_in) {
path: cpp/core.cc
user: login: andychu
body: The param name should match Python, so it should be something like  `fd_list`

created_at: 2025-03-30T16:41:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -254,6 +254,27 @@ bool InputAvailable(int fd) {
   return select(FD_SETSIZE, &fds, NULL, NULL, &timeout) > 0;
 }
 
+List<int>* WaitForInputs(List<int>* fds_in) {
+  auto* ret = NewList<int>();
+  fd_set fds;
+  FD_ZERO(&fds);
path: cpp/core.cc
user: login: andychu
body: `fds` could be something like `fd_bits`, to distinguish from `fd_list`

It is a bit mask passed to the kernel


created_at: 2025-03-30T16:42:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,9 +963,30 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
     }
 
+### captureOutputs()
+
+Capture stdout and stderr of a command a string.
path: doc/ref/chap-type-method.md
user: login: andychu
body: - typo "a string"
- should mentino status



created_at: 2025-03-30T16:42:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,9 +963,30 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
     }
 
+### captureOutputs()
+
+Capture stdout and stderr of a command a string.
+
+    var c = ^(echo hi; echo hello >&2;)
+    var outputs = _io.captureOutputs(c)  # => { stdout: "hi", stderr: "hello", status: 0 }
+
+It's like `io.captureStdout` but captures both stdout and stderr.  Trailing newlines `\n` are
path: doc/ref/chap-type-method.md
user: login: andychu
body: We may want to keep trailing newlines?

The only reason captureStdout() does it is because that's how shell command sub work

I also wonder if there are better names for `captureOutputs()` ...  I was thinking `captureStdoutStderr()`, but that leaves out `status`

or `capture3()` or something

created_at: 2025-03-30T16:44:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -92,6 +92,9 @@ proc my-cd (; b) {
   # Yup, this is a problem
   var s = io.captureStdout(b)
   echo "stdout $s"
+
+  setvar s = io.captureOutputs(b)
+  echo "stdout $[s.stdout]"
path: spec/ysh-blocks.test.sh
user: login: andychu
body: this is a bit confusing because it doesn't change any of the assertions ?  what happened?

created_at: 2025-03-30T16:45:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,9 +963,30 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
     }
 
+### captureOutputs()
+
+Capture stdout and stderr of a command a string.
+
+    var c = ^(echo hi; echo hello >&2;)
+    var outputs = _io.captureOutputs(c)  # => { stdout: "hi", stderr: "hello", status: 0 }
+
+It's like `io.captureStdout` but captures both stdout and stderr.  Trailing newlines `\n` are
path: doc/ref/chap-type-method.md
user: login: Melkor333
body: I thought especially of `capture3`, too. But it's IMO a bit too "bound to the output". E.g. if we ever want to add something like `additional_fds=[3, 4]`, or execution time, etc. Therefore `Outputs` seemed the cleanest.
created_at: 2025-04-01T19:56:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,9 +963,30 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
     }
 
+### captureOutputs()
+
+Capture stdout and stderr of a command a string.
+
+    var c = ^(echo hi; echo hello >&2;)
+    var outputs = _io.captureOutputs(c)  # => { stdout: "hi", stderr: "hello", status: 0 }
+
+It's like `io.captureStdout` but captures both stdout and stderr.  Trailing newlines `\n` are
path: doc/ref/chap-type-method.md
user: login: andychu
body: I also thought it could be `captureStd()` or `captureAll()`

I feel like captureStdout() + captureOutputs() is a tiny bit awkward, because is "stderr" and "status" an "output"?  It is hard to find a good name though

I guess All() can also refer to the fact that it doesn't strip newlines

created_at: 2025-04-02T02:31:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,9 +963,30 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
     }
 
+### captureOutputs()
+
+Capture stdout and stderr of a command a string.
+
+    var c = ^(echo hi; echo hello >&2;)
+    var outputs = _io.captureOutputs(c)  # => { stdout: "hi", stderr: "hello", status: 0 }
+
+It's like `io.captureStdout` but captures both stdout and stderr.  Trailing newlines `\n` are
path: doc/ref/chap-type-method.md
user: login: Melkor333
body: hm `captureAll` sounds good! Let me change it to that and fix the remaining points.
created_at: 2025-04-02T04:35:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -632,6 +639,64 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def CaptureOutputs(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        eofcount = 0
+        finished = False
+        while True:
+            fds = pyos.WaitForInputs([stdout_fd, stderr_fd])
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    eofcount += 1
+                if eofcount > 1:
path: core/executor.py
user: login: Melkor333
body: LOL I ran right into that assert. If a FD returns EOF I shouldn't check for it anymore 😄
created_at: 2025-04-02T06:03:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -92,6 +92,9 @@ proc my-cd (; b) {
   # Yup, this is a problem
   var s = io.captureStdout(b)
   echo "stdout $s"
+
+  setvar s = io.captureOutputs(b)
+  echo "stdout $[s.stdout]"
path: spec/ysh-blocks.test.sh
user: login: Melkor333
body: Soo turns out this test was marked as "allowed failure" because it didn't work in the past. But it does work now, so I added the proper `STDOUT`. I think it's behaviour is correct now.
created_at: 2025-04-02T06:22:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -232,6 +232,29 @@ def Call(self, rd):
         return value.Str(stdout_str)
 
 
+class CaptureOutputs(vm._Callable):
path: builtin/method_io.py
user: login: andychu
body: this should be CaptureAll() now too

created_at: 2025-04-03T16:57:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -272,6 +272,13 @@ def CaptureStdout(self, node):
         """
         return 0, ''
 
+    def CaptureOutputs(self, node):
path: core/executor.py
user: login: andychu
body: I guess this should be CaptureAll() too, or maybe Capture3() since it's internal ?    


created_at: 2025-04-03T16:57:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -278,10 +278,18 @@ def InputAvailable(fd):
     # type: (int) -> bool
     # similar to lib/sh/input_avail.c in bash
path: core/pyos.py
user: login: andychu
body: remove this comment, I think it was copied and pasted from InputAvailable

created_at: 2025-04-03T16:58:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -278,10 +278,18 @@ def InputAvailable(fd):
     # type: (int) -> bool
     # similar to lib/sh/input_avail.c in bash
     # read, write, except
-    r, w, exc = select.select([fd], [], [fd], 0)
+    r, _, _ = select.select([fd], [], [], 0)
     return len(r) != 0
 
 
+def WaitForReading(fd_list):
+    # type: (List[int]) -> List[int]
+    # Get a list of fds and return fds which have available input
path: core/pyos.py
user: login: andychu
body: I would turn this comment into a docstring, or remove it

created_at: 2025-04-03T16:59:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -632,6 +639,62 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def CaptureOutputs(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        open_fds = [stdout_fd, stderr_fd]
+        while True:
+            fds = pyos.WaitForReading(open_fds)
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    open_fds.remove(fd)
+
+            if len(open_fds) == 0:
path: core/executor.py
user: login: andychu
body: I am not sure we need both `break` conditions ... but I guess it avoids calling select() with an empty list, so maybe that's useful

created_at: 2025-04-03T17:00:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -963,7 +963,27 @@ If the command fails, `captureStdout()` raises an error, which can be caught
 with `try`.
 
     try {
-      var s = _io->captureStdout(c)
+      var s = io.captureStdout(c)
+    }
+
+### captureAll()
+
+Capture stdout and stderr and status of a command.
+
+    var c = ^(echo hi; echo hello >&2;)
+    var outputs = io.captureAll(c)  # => { stdout: "hi", stderr: "hello", status: 0 }
+
+It's vaguely similar to `io.captureStdout` but captures both stdout, stderr into a `dict`.
path: doc/ref/chap-type-method.md
user: login: andychu
body: I would remove the word "vaguely"

And capitalize `Dict`, which is a type

created_at: 2025-04-03T17:01:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -632,6 +639,62 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def CaptureOutputs(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        open_fds = [stdout_fd, stderr_fd]
+        while True:
+            fds = pyos.WaitForReading(open_fds)
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    open_fds.remove(fd)
+
+            if len(open_fds) == 0:
path: core/executor.py
user: login: Melkor333
body: According to [stackoverflow](https://stackoverflow.com/questions/15109527/return-value-of-select-with-empty-sets):

>     Some code calls select() with all three sets empty, nfds zero, and a non-NULL timeout as a fairly portable way to sleep with subsecond precision.

I have the feeling we might get an indefinite sleep without the second break...
created_at: 2025-04-05T08:15:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -632,6 +639,62 @@ def CaptureStdout(self, node):
 
         return status, stdout_str
 
+    def CaptureOutputs(self, node):
+        # type: (command_t) -> Tuple[int, str, str]
+
+        p = self._MakeProcess(node, self.exec_opts.inherit_errexit(),
+                              self.exec_opts.errtrace())
+        # Shell quirk: Command subs remain part of the shell's process group, so we
+        # don't use p.AddStateChange(process.SetPgid(...))
+
+        stdout_fd, w = posix.pipe()
+        stderr_fd, w2 = posix.pipe()
+        p.AddStateChange(process.StdoutToPipe(stdout_fd, w))
+        p.AddStateChange(process.StderrToPipe(stderr_fd, w2))
+
+        p.StartProcess(trace.CommandSub)
+        #log('Command sub started %d', pid)
+
+        stdout_chunks = []  # type: List[str]
+        stderr_chunks = []  # type: List[str]
+        posix.close(w)  # not going to write
+        posix.close(w2)  # not going to write
+        open_fds = [stdout_fd, stderr_fd]
+        while True:
+            fds = pyos.WaitForReading(open_fds)
+
+            # zero outputs mean something went wrong
+            if len(fds) == 0:
+                break
+
+            for fd in fds:
+                if fd == stdout_fd:
+                    n, err_num = pyos.Read(fd, 4096, stdout_chunks)
+                else:
+                    n, err_num = pyos.Read(fd, 4096, stderr_chunks)
+                if n < 0:
+                    if err_num == EINTR:
+                        pass  # retry
+                    else:
+                        # Like the top level IOError handler
+                        e_die_status(
+                            2,
+                            'Oils I/O error (read): %s' % posix.strerror(err_num))
+                elif n == 0:  # EOF
+                    open_fds.remove(fd)
+
+            if len(open_fds) == 0:
path: core/executor.py
user: login: andychu
body: Ah OK good point, thanks for looking that up

created_at: 2025-04-05T17:39:47Z
author_association: CONTRIBUTOR

PR comments:
user: login: Melkor333
created_at: 2025-03-27T22:02:50Z
body: @andychu it compiles now but gives an indexError I don't really understand...
```
ysh-0.28$ var x = io.captureStdout(^(echo hi; echo hi>&2;))                         
libc++abi: terminating due to uncaught exception of type IndexError*
Aborted
```
Is it "easy" to compile the cpp tarball to throw nice errors with more info?
Also, do you think this `WaitForInputs` function (also the cpp version) makes roughly sense? Or would you do it completely differently?
author_association: CONTRIBUTOR
user: login: Melkor333
created_at: 2025-03-28T12:46:54Z
body: I've done a first implementation of `io.captureOutputs` 🥳 
BUT the error handling needs a bit of "fleshing out". It does add a `status` but fails when `fail=false` is not passed as an extra argument. I wonder if the `status` should NOT be added to the dict when `fail=true` (default), as it will implicitly always be `0`.

Also. should this `fail` option be added to `captureStdout` as well? Though we return a string on that function and I don't think we want to change that!

Anyway one issue I have (and why the tests are failing) is that when the command is executed and FAILs, the "Oils error" `X command.Simple failed with status Y` is printed to stderr and thus added to the `stderr` variable:
```
test/spec.sh ysh-method-io -v
        our_shell           : ysh
        oils_failures_allowed: 0

ysh-method-io.test.sh
case    line    ysh
  0       4     pass    captureStdout() is like $()
  1      20     pass    captureStdout() failure
  2      40     pass    io->eval() is like eval builtin
  3      54     pass    captureOutputs() is like $() but captures stderr as well
  4      70     pass    captureOutputs() failure
  5      90     FAIL    captureOutputs() with fail=false
case: 5

[ysh stdout]
Expected '(Dict)   {"code":0}\n(Dict)   {"stdout":"one","stderr":"","status":1}\n'
Got      '(Dict)   {"code":0}\n(Dict)   {"stdout":"one","stderr":"  var c = ^(echo one; false; echo two)\\n                      ^~~~~\\n[ stdin ]:1: errexit PID 2185909: command.Simple failed with status 1","status":1}\n'

ysh stdout:
(Dict)   {"code":0}
(Dict)   {"stdout":"one","stderr":"  var c = ^(echo one; false; echo two)\n                      ^~~~~\n[ stdin ]:1: errexit PID 2185909: command.Simple failed with status 1","status":1}

ysh stderr:


  6     110     pass    io->eval() with failing command - caller must handle
  7     128     pass    io->eval() with exit

                ysh
        pass    7
        FAIL    1
        total   8

ysh-method-io: FATAL: Got 1 Oils failures, but 0 are allowed
ysh-method-io: FATAL: Got 1 total failures, but 0 are allowed
```

Is there a way to not have this error message printed? I suspect it has something to do with `p.StartProcess(trace.CommandSub)` that might need to be changed. 🤔 From taking a look at `core/process.py` it seems like there is currently no way to disable the "oils error", but I could be wrong.
author_association: CONTRIBUTOR
user: login: andychu
created_at: 2025-03-28T18:05:49Z
body: Is it correct to ignore that text on stderr?

Generally the stderr of the shell interpreter itself is  fair game, e.g. in shell:

```
$ x=$(declare bad-name 2>&1 ); echo "x=[$x]"
x=[-bash: declare: `bad-name': not a valid identifier]
```



We asked for stderr there, so we got stderr

there is a `shopt --unset verbose_errexit` option I believe

author_association: CONTRIBUTOR
user: login: Melkor333
created_at: 2025-03-28T21:39:29Z
body: > Is it correct to ignore that text on stderr?
> 
> Generally the stderr of the shell interpreter itself is  fair game, e.g. in shell:
> 
> ```
> $ x=$(declare bad-name 2>&1 ); echo "x=[$x]"
> x=[-bash: declare: `bad-name': not a valid identifier]
> ```
> 
> 
> 
> We asked for stderr there, so we got stderr
> 
> there is a `shopt --unset verbose_errexit` option I believe
> 

Ah, I thought this message is also printed when an executable fails, but it's not (I should do less believing and more testing!). I'll just make the test case consistent then.
Do you think there's something else to consider or  do you think it's good when the spec test succeeds?
author_association: CONTRIBUTOR
user: login: Melkor333
created_at: 2025-03-29T19:48:32Z
body: @andychu everything done now. :-)
author_association: CONTRIBUTOR
user: login: andychu
created_at: 2025-03-30T16:47:26Z
body: and yeah I think we can probably leave out `fail=true`?  If we don't have a use case for that, we can leave it out

And maybe don't strip newlines


author_association: CONTRIBUTOR
user: login: Melkor333
created_at: 2025-04-02T06:49:56Z
body: @andychu I don't get why the CI fails. Seems like something unrelated?
author_association: CONTRIBUTOR
user: login: andychu
created_at: 2025-04-03T02:50:05Z
body: Hmm yeah we started getting this:

```
mkdir: created directory '_tmp/code-blocks/doc'
mkdir: created directory '_tmp/code-blocks/doc/ref'
Traceback (most recent call last):
  File "doctools/cmark.py", line 65, in <module>
    raise AssertionError("Couldn't find libcmark.so")
AssertionError: Couldn't find libcmark.so
```

which is unrelated to your change

That is odd, I didn't change anything ... looking into it

author_association: CONTRIBUTOR
user: login: andychu
created_at: 2025-04-03T16:56:59Z
body: OK I resolved the CI error, let me take another look


author_association: CONTRIBUTOR
user: login: Melkor333
created_at: 2025-04-18T19:35:22Z
body: @andychu sorry for the delay. finally finished the cosmetics.

btw. I didn't intend to fix the `shell.nix` but I had this minor change still configured and it just worked when I briefly tested it today. Reverting my change broke it again, despite that it should work with a more recent libreadline. It might be an inconsistency in the nix build working half of the time (which would be super odd).
author_association: CONTRIBUTOR
user: login: andychu
created_at: 2025-04-21T14:43:04Z
body: Merged, thank you!
author_association: CONTRIBUTOR