Repository: xonsh/xonsh

Issue:
Universal new lines in captured object output
Hi! The snail-hole went straight on like a tunnel for some way, and then dipped suddenly down, so suddenly that Alice had not a moment to think about stopping herself before she found herself falling down a very deep conch.

While I'm working on [XEP-2](https://github.com/anki-code/xonsh-operators-proposal/blob/main/XEP-2.rst) I've found that `!().out` returns not original command output. Example:

```python
echo -n '1\n2\r\r3\r\n4\n' > txt
!(head -n 10 txt).out
# '1\n2\r\r3\n4\n'

# Prepared by xontrib-hist-format
```
Here we've got `3\n4` instead of `3\r\n4`. This behavior with `!().out` is expected or it's the bug?

## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**



Discussion:

Resolving patch:
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -217,14 +217,29 @@ def ensure_list_of_strs(x):
     return rtn
 
 
-def list_of_strs_or_callables(x):
-    """Ensures that x is a list of strings or functions"""
+def ensure_str_or_callable(x):
+    """Ensures that x is single string or function."""
     if isinstance(x, str) or callable(x):
-        rtn = [x]
+        return x
+    if isinstance(x, bytes):
+        # ``os.fsdecode`` decodes using "surrogateescape" on linux and "strict" on windows.
+        # This is used to decode bytes for interfacing with the os, notably for command line arguments.
+        # See https://www.python.org/dev/peps/pep-0383/#specification
+        return os.fsdecode(x)
+    return str(x)
+
+
+def list_of_strs_or_callables(x):
+    """
+    Ensures that x is a list of strings or functions.
+    This is called when using the ``@()`` operator to expand it's content.
+    """
+    if isinstance(x, (str, bytes)) or callable(x):
+        rtn = [ensure_str_or_callable(x)]
     elif isinstance(x, cabc.Iterable):
-        rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x]
+        rtn = list(map(ensure_str_or_callable, x))
     else:
-        rtn = [str(x)]
+        rtn = [ensure_str_or_callable(x)]
     return rtn
 
 
diff --git a/xonsh/procs/pipelines.py b/xonsh/procs/pipelines.py
--- a/xonsh/procs/pipelines.py
+++ b/xonsh/procs/pipelines.py
@@ -147,6 +147,7 @@ def __init__(self, specs):
         self.input = self._output = self.errors = self.endtime = None
         self._closed_handle_cache = {}
         self.lines = []
+        self._raw_output = self._raw_error = b""
         self._stderr_prefix = self._stderr_postfix = None
         self.term_pgid = None
 
@@ -331,6 +332,7 @@ def tee_stdout(self):
         enc = env.get("XONSH_ENCODING")
         err = env.get("XONSH_ENCODING_ERRORS")
         lines = self.lines
+        raw_out_lines = []
         stream = self.captured not in STDOUT_CAPTURE_KINDS
         if stream and not self.spec.stdout:
             stream = False
@@ -346,6 +348,8 @@ def tee_stdout(self):
                 else:
                     sys.stdout.write(line.decode(encoding=enc, errors=err))
                 sys.stdout.flush()
+            # save the raw bytes
+            raw_out_lines.append(line)
             # do some munging of the line before we return it
             if line.endswith(crnl):
                 line = line[:-2] + nl
@@ -357,6 +361,9 @@ def tee_stdout(self):
             lines.append(line)
             yield line
 
+        # using join is more efficient than concatenating in a loop
+        self._raw_output = b"".join(raw_out_lines)
+
     def stream_stderr(self, lines):
         """Streams lines to sys.stderr and the errors attribute."""
         if not lines:
@@ -376,6 +383,8 @@ def stream_stderr(self, lines):
         else:
             sys.stderr.write(b.decode(encoding=enc, errors=err))
         sys.stderr.flush()
+        # save the raw bytes
+        self._raw_error = b
         # do some munging of the line before we save it to the attr
         b = b.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
         b = RE_HIDE_ESCAPE.sub(b"", b)
@@ -613,6 +622,18 @@ def err(self):
         self.end()
         return self.errors
 
+    @property
+    def raw_out(self):
+        """Output as raw bytes."""
+        self.end()
+        return self._raw_output
+
+    @property
+    def raw_err(self):
+        """Errors as raw bytes."""
+        self.end()
+        return self._raw_error
+
     @property
     def pid(self):
         """Process identifier."""


Test patch:
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
new file mode 100644
--- /dev/null
+++ b/tests/test_pipelines.py
@@ -0,0 +1,61 @@
+"""
+Tests for command pipelines.
+"""
+
+import os
+import pytest
+import builtins
+
+from xonsh.platform import ON_WINDOWS
+from xonsh.procs.pipelines import CommandPipeline
+from tests.tools import skip_if_on_windows, skip_if_on_unix
+
+
+@pytest.fixture(autouse=True)
+def patched_events(monkeypatch, xonsh_events, xonsh_execer):
+    # needed for ci tests
+    monkeypatch.setattr('builtins.events', xonsh_events, raising=False)
+    monkeypatch.setitem(builtins.__xonsh__.env, 'RAISE_SUBPROC_ERROR', False)  # for the failing `grep` commands
+    if ON_WINDOWS:
+        monkeypatch.setattr('builtins.aliases', {
+            "echo": "cmd /c echo".split(),
+            "grep": "cmd /c findstr".split(),
+        }, raising=False)
+
+
+@pytest.mark.parametrize("cmdline, stdout, stderr", (
+        ("!(echo hi)", "hi\n", ""),
+        ("!(echo hi o>e)", "", "hi\n"),
+
+        pytest.param("![echo hi]", "hi\n", "", marks=pytest.mark.xfail(
+            ON_WINDOWS, reason="ConsoleParallelReader doesn't work without a real console")),
+        pytest.param("![echo hi o>e]", "", "hi\n", marks=pytest.mark.xfail(
+            ON_WINDOWS, reason="stderr isn't captured in ![] on windows")),
+
+        pytest.param(r"!(echo 'hi\nho')", "hi\nho\n", "", marks=skip_if_on_windows),  # won't work with cmd
+        # for some reason cmd's echo adds an extra space:
+        pytest.param(r"!(cmd /c 'echo hi && echo ho')", "hi \nho\n", "", marks=skip_if_on_unix),
+
+        ("!(echo hi | grep h)", "hi\n", ""),
+        ("!(echo hi | grep x)", "", ""),
+))
+def test_command_pipeline_capture(cmdline, stdout, stderr, xonsh_execer):
+    pipeline: CommandPipeline = xonsh_execer.eval(cmdline)
+    assert pipeline.out == stdout
+    assert pipeline.err == (stderr or None)
+    assert pipeline.raw_out == stdout.replace("\n", os.linesep).encode()
+    assert pipeline.raw_err == stderr.replace("\n", os.linesep).encode()
+
+
+@pytest.mark.parametrize("cmdline, output", (
+        ("echo hi", "hi\n"),
+        ("echo hi | grep h", "hi\n"),
+        ("echo hi | grep x", ""),
+        pytest.param("echo -n hi", "hi", marks=skip_if_on_windows),
+))
+def test_simple_capture(cmdline, output, xonsh_execer):
+    assert xonsh_execer.eval(f"$({cmdline})") == output
+
+
+def test_raw_substitution(xonsh_execer):
+    assert xonsh_execer.eval("$(echo @(b'bytes!'))") == "bytes!\n"