Repository: PrefectHQ/prefect

Issue:
shellTask logs all stdout when `ShellTask(return_all=True)`
**Context**
Currently, when `prefect.tasks.shell.ShellTask` is invoked with `return_all=True`, the shell task will not log the stdout through logger.debug.

This is defined in doc 
https://docs.prefect.io/api/latest/tasks/shell.html#shelltask
and implement in the code.
https://github.com/PrefectHQ/prefect/blob/73fe1eb57945fc13bdd36b361a6a17c3f324b1ba/src/prefect/tasks/shell.py#L107-L111

**Problem**

Sometimes, we execute a long-running shell process, for example, we are running dbt, which might take 20 to 40 minutes to run. the task would need to return the entire stdout log as its output, for the next task to parse and interpret the result. Meanwhile, We would also like to see the shell stdout logging out in real-time, which makes it easy for us to inspect issues and monitoring / debug shell runs on-the-go, instead of waiting for 20 minutes not knowing what is going on.

**proposed change**

```
                for raw_line in iter(sub_process.stdout.readline, b""):
                    line = raw_line.decode("utf-8").rstrip()
                    if self.return_all:
                        lines.append(line)
                    # we still log every line
                    self.logger.debug(line)
```


Discussion:
I think this is reasonable although it may be better implemented by adding a `stream_logs: bool = True` and having this behavior be entirely independent of `return_all`. @joshmeek would we be willing to accept a PR for this?
yes, `stream_logs: bool = True` would be even better.
Yeah I'm game for a steam logs kwarg!
We do need this fairly quickly. Please let me know how I could contribute to any. 

Resolving patch:
diff --git a/src/prefect/tasks/shell.py b/src/prefect/tasks/shell.py
--- a/src/prefect/tasks/shell.py
+++ b/src/prefect/tasks/shell.py
@@ -1,7 +1,7 @@
 import os
 import tempfile
 from subprocess import PIPE, STDOUT, Popen
-from typing import Any
+from typing import Any, List, Union, Optional
 
 import prefect
 from prefect.utilities.tasks import defaults_from_attrs
@@ -11,6 +11,9 @@ class ShellTask(prefect.Task):
     """
     Task for running arbitrary shell commands.
 
+    NOTE: This task combines stderr and stdout because reading from both
+          streams without blocking is tricky.
+
     Args:
         - command (string, optional): shell command to be executed; can also be
             provided post-initialization by calling this task instance
@@ -24,9 +27,13 @@ class ShellTask(prefect.Task):
         - return_all (bool, optional): boolean specifying whether this task
             should return all lines of stdout as a list, or just the last line
             as a string; defaults to `False`
-        - log_stderr (bool, optional): boolean specifying whether this task
-            should log the output from stderr in the case of a non-zero exit code;
-            defaults to `False`
+        - log_stderr (bool, optional): boolean specifying whether this task should log
+            the output in the case of a non-zero exit code; defaults to `False`. This
+             actually logs both stderr and stdout and will only log the last line of
+             output unless `return_all` is `True`
+        - stream_output (bool, optional): specifies whether this task should log
+            the output as it occurs. If enabled, `log_stderr` will be ignored as the
+            output will have already been logged. defaults to `False`
         - **kwargs: additional keyword arguments to pass to the Task constructor
 
     Example:
@@ -52,6 +59,7 @@ def __init__(
         shell: str = "bash",
         return_all: bool = False,
         log_stderr: bool = False,
+        stream_output: bool = False,
         **kwargs: Any
     ):
         self.command = command
@@ -60,10 +68,13 @@ def __init__(
         self.shell = shell
         self.return_all = return_all
         self.log_stderr = log_stderr
+        self.stream_output = stream_output
         super().__init__(**kwargs)
 
     @defaults_from_attrs("command", "env")
-    def run(self, command: str = None, env: dict = None) -> str:
+    def run(
+        self, command: str = None, env: dict = None
+    ) -> Optional[Union[str, List[str]]]:
         """
         Run the shell command.
 
@@ -76,8 +87,8 @@ def run(self, command: str = None, env: dict = None) -> str:
                 the subprocess
 
         Returns:
-            - stdout (string): if `return_all` is `False` (the default), only
-                the last line of stdout is returned, otherwise all lines are
+            - output (string): if `return_all` is `False` (the default), only
+                the last line of output is returned, otherwise all lines are
                 returned, which is useful for passing result of shell command
                 to other downstream tasks. If there is no output, `None` is
                 returned.
@@ -100,27 +111,31 @@ def run(self, command: str = None, env: dict = None) -> str:
             with Popen(
                 [self.shell, tmp.name], stdout=PIPE, stderr=STDOUT, env=current_env
             ) as sub_process:
-                lines = []
                 line = None
+                lines = []
                 for raw_line in iter(sub_process.stdout.readline, b""):
                     line = raw_line.decode("utf-8").rstrip()
+
                     if self.return_all:
                         lines.append(line)
-                    else:
-                        # if we're returning all, we don't log every line
+
+                    if self.stream_output:
                         self.logger.debug(line)
+
                 sub_process.wait()
                 if sub_process.returncode:
                     msg = "Command failed with exit code {}".format(
                         sub_process.returncode,
                     )
+
                     self.logger.error(msg)
 
-                    if self.log_stderr:
-                        self.logger.error("\n".join(lines))
+                    if self.log_stderr and not self.stream_output:
+                        self.logger.error("\n".join(lines) if self.return_all else line)
+
+                    raise prefect.engine.signals.FAIL(
+                        msg,
+                        result=lines if self.return_all else line,
+                    )
 
-                    raise prefect.engine.signals.FAIL(msg) from None  # type: ignore
-        if self.return_all:
-            return lines
-        else:
-            return line
+        return lines if self.return_all else line


Test patch:
diff --git a/tests/tasks/test_shell.py b/tests/tasks/test_shell.py
--- a/tests/tasks/test_shell.py
+++ b/tests/tasks/test_shell.py
@@ -108,6 +108,34 @@ def test_shell_logs_non_zero_exit(caplog):
     assert "Command failed" in error_log[0].message
 
 
+def test_shell_attaches_result_to_failure(caplog):
+    def assert_fail_result(task, state):
+        assert (
+            state.result == "foo"
+            "ls: surely_a_dir_that_doesnt_exist: No such file or directory"
+        )
+
+    with Flow(name="test") as f:
+        task = ShellTask(on_failure=assert_fail_result)(
+            command="echo foo && ls surely_a_dir_that_doesnt_exist"
+        )
+    out = f.run()
+    assert out.is_failed()
+
+
+@pytest.mark.parametrize("stream_output", [True, False])
+def test_shell_respects_stream_output(caplog, stream_output):
+
+    with Flow(name="test") as f:
+        ShellTask(stream_output=stream_output)(
+            command="echo foo && echo bar",
+        )
+    f.run()
+
+    stdout_in_log = "foo" in caplog.messages and "bar" in caplog.messages
+    assert stdout_in_log == stream_output
+
+
 def test_shell_logs_stderr_on_non_zero_exit(caplog):
     with Flow(name="test") as f:
         task = ShellTask(log_stderr=True, return_all=True)(