Repository: sqlfluff/sqlfluff

Issue:
Write-output human format does not produce result
### Search before asking

- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.


### What Happened

When running SQLFluff using the following statement:
`python -m sqlfluff lint --write-output test.txt --config=config/sql-lint.cfg`
no result was produced.

### Expected Behaviour

I expect a file to appear, in this case called test,txt, containing all violations found.

### Observed Behaviour

Looking through the code I saw human was the default format so expected adding --format=human would not make a difference. To be sure, I also ran the statement using the flag and it still produced nothing.

To make sure it was just the human format which was having problems, I also executed the statement using --format=json,yaml,github-annotations, all of which did produce the expected result which leads me to believe there is something wrong with the human format.

### How to reproduce

I imagine simply executing `sqlfluff lint --write-output test.txt example.sql`

### Dialect

T-SQL

### Version

0.11.2

### Configuration

[sqlfluff]
dialect = tsql
exclude_rules = L014,
                L016,
                L031,
                L035,
                L059

### Are you willing to work on and submit a PR to address the issue?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)



Discussion:
This line in cli.py seems largely to blame -- it somewhat conflates output _format_ with writing to a file or not.
```
non_human_output = (format != FormatType.human.value) or (write_output is not None)
```

It will require some care to fix this. Simply removing `or (write_output is not None)` didn't seem to fix it.

As a workaround until this is fixed, you may be able to use output redirection, e.g.
```
python -m sqlfluff lint --config=config/sql-lint.cfg > test.txt
```


Your workaround does work for me, thank you. Seeing as this solution is only a workaround I imagine closing the ticket is not preferable.

Patch:
diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py
--- a/src/sqlfluff/cli/commands.py
+++ b/src/sqlfluff/cli/commands.py
@@ -1,6 +1,7 @@
 """Contains the CLI."""
 
 from itertools import chain
+import os
 import sys
 import json
 import logging
@@ -29,9 +30,10 @@
     colorize,
     format_dialect_warning,
     format_dialects,
-    CallbackFormatter,
+    OutputStreamFormatter,
 )
 from sqlfluff.cli.helpers import cli_table, get_package_version
+from sqlfluff.cli.outputstream import make_output_stream, OutputStream
 
 # Import from sqlfluff core.
 from sqlfluff.core import (
@@ -332,28 +334,9 @@ def get_config(
         sys.exit(66)
 
 
-def _callback_handler(cfg: FluffConfig) -> Callable:
-    """Returns function which will be bound as a callback for printing passed message.
-
-    Called in `get_linter_and_formatter`.
-    """
-
-    def _echo_with_tqdm_lock(message: str) -> None:
-        """Makes sure that message printing (echoing) will be not in conflict with tqdm.
-
-        It may happen that progressbar conflicts with extra printing. Nothing very
-        serious happens then, except that there is printed (not removed) progressbar
-        line. The `external_write_mode` allows to disable tqdm for writing time.
-        """
-        with tqdm.external_write_mode():
-            click.echo(message=message, color=cfg.get("color"))
-
-    return _echo_with_tqdm_lock
-
-
 def get_linter_and_formatter(
-    cfg: FluffConfig, silent: bool = False
-) -> Tuple[Linter, CallbackFormatter]:
+    cfg: FluffConfig, output_stream: Optional[OutputStream] = None
+) -> Tuple[Linter, OutputStreamFormatter]:
     """Get a linter object given a config."""
     try:
         # We're just making sure it exists at this stage.
@@ -364,20 +347,12 @@ def get_linter_and_formatter(
     except KeyError:  # pragma: no cover
         click.echo(f"Error: Unknown dialect '{cfg.get('dialect')}'")
         sys.exit(66)
-
-    if not silent:
-        # Instantiate the linter and return it (with an output function)
-        formatter = CallbackFormatter(
-            callback=_callback_handler(cfg=cfg),
-            verbosity=cfg.get("verbose"),
-            output_line_length=cfg.get("output_line_length"),
-        )
-        return Linter(config=cfg, formatter=formatter), formatter
-    else:
-        # Instantiate the linter and return. NB: No formatter
-        # in the Linter and a black formatter otherwise.
-        formatter = CallbackFormatter(callback=lambda m: None, verbosity=0)
-        return Linter(config=cfg), formatter
+    formatter = OutputStreamFormatter(
+        output_stream=output_stream or make_output_stream(cfg),
+        verbosity=cfg.get("verbose"),
+        output_line_length=cfg.get("output_line_length"),
+    )
+    return Linter(config=cfg, formatter=formatter), formatter
 
 
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
@@ -521,7 +496,8 @@ def lint(
     )
     non_human_output = (format != FormatType.human.value) or (write_output is not None)
     file_output = None
-    lnt, formatter = get_linter_and_formatter(config, silent=non_human_output)
+    output_stream = make_output_stream(config, format, write_output)
+    lnt, formatter = get_linter_and_formatter(config, output_stream)
 
     verbose = config.get("verbose")
     progress_bar_configuration.disable_progress_bar = disable_progress_bar
@@ -610,6 +586,7 @@ def lint(
     if file_output:
         dump_file_payload(write_output, cast(str, file_output))
 
+    output_stream.close()
     if bench:
         click.echo("==== overall timings ====")
         click.echo(cli_table([("Clock time", result.total_time)]))
@@ -733,7 +710,10 @@ def fix(
         extra_config_path, ignore_local_config, require_dialect=False, **kwargs
     )
     fix_even_unparsable = config.get("fix_even_unparsable")
-    lnt, formatter = get_linter_and_formatter(config, silent=fixing_stdin)
+    output_stream = make_output_stream(
+        config, None, os.devnull if fixing_stdin else None
+    )
+    lnt, formatter = get_linter_and_formatter(config, output_stream)
 
     verbose = config.get("verbose")
     progress_bar_configuration.disable_progress_bar = disable_progress_bar
@@ -974,7 +954,8 @@ def parse(
     # We don't want anything else to be logged if we want json or yaml output
     # unless we're writing to a file.
     non_human_output = (format != FormatType.human.value) or (write_output is not None)
-    lnt, formatter = get_linter_and_formatter(c, silent=non_human_output)
+    output_stream = make_output_stream(c, format, write_output)
+    lnt, formatter = get_linter_and_formatter(c, output_stream)
     verbose = c.get("verbose")
     recurse = c.get("recurse")
 
@@ -1020,7 +1001,7 @@ def parse(
         # iterative print for human readout
         if format == FormatType.human.value:
             violations_count = _print_out_violations_and_timing(
-                bench, code_only, total_time, verbose, parsed_strings
+                output_stream, bench, code_only, total_time, verbose, parsed_strings
             )
         else:
             parsed_strings_dict = [
@@ -1072,6 +1053,7 @@ def parse(
 
 
 def _print_out_violations_and_timing(
+    output_stream: OutputStream,
     bench: bool,
     code_only: bool,
     total_time: float,
@@ -1086,30 +1068,30 @@ def _print_out_violations_and_timing(
         timing.add(parsed_string.time_dict)
 
         if parsed_string.tree:
-            click.echo(parsed_string.tree.stringify(code_only=code_only))
+            output_stream.write(parsed_string.tree.stringify(code_only=code_only))
         else:
             # TODO: Make this prettier
-            click.echo("...Failed to Parse...")  # pragma: no cover
+            output_stream.write("...Failed to Parse...")  # pragma: no cover
 
         violations_count += len(parsed_string.violations)
         if parsed_string.violations:
-            click.echo("==== parsing violations ====")  # pragma: no cover
+            output_stream.write("==== parsing violations ====")  # pragma: no cover
         for v in parsed_string.violations:
-            click.echo(format_violation(v))  # pragma: no cover
+            output_stream.write(format_violation(v))  # pragma: no cover
         if parsed_string.violations and parsed_string.config.get("dialect") == "ansi":
-            click.echo(format_dialect_warning())  # pragma: no cover
+            output_stream.write(format_dialect_warning())  # pragma: no cover
 
         if verbose >= 2:
-            click.echo("==== timings ====")
-            click.echo(cli_table(parsed_string.time_dict.items()))
+            output_stream.write("==== timings ====")
+            output_stream.write(cli_table(parsed_string.time_dict.items()))
 
     if verbose >= 2 or bench:
-        click.echo("==== overall timings ====")
-        click.echo(cli_table([("Clock time", total_time)]))
+        output_stream.write("==== overall timings ====")
+        output_stream.write(cli_table([("Clock time", total_time)]))
         timing_summary = timing.summary()
         for step in timing_summary:
-            click.echo(f"=== {step} ===")
-            click.echo(cli_table(timing_summary[step].items()))
+            output_stream.write(f"=== {step} ===")
+            output_stream.write(cli_table(timing_summary[step].items()))
 
     return violations_count
 
diff --git a/src/sqlfluff/cli/formatters.py b/src/sqlfluff/cli/formatters.py
--- a/src/sqlfluff/cli/formatters.py
+++ b/src/sqlfluff/cli/formatters.py
@@ -2,7 +2,7 @@
 
 
 from io import StringIO
-from typing import Callable, List, Union
+from typing import List, Union
 
 from sqlfluff.cli.helpers import (
     colorize,
@@ -12,6 +12,7 @@
     get_python_implementation,
     pad_line,
 )
+from sqlfluff.cli.outputstream import OutputStream
 from sqlfluff.core import SQLBaseError, FluffConfig, Linter
 from sqlfluff.core.enums import Color
 from sqlfluff.core.linter import LintedFile
@@ -199,8 +200,8 @@ def format_dialect_warning():  # pragma: no cover
     )
 
 
-class CallbackFormatter:
-    """Formatter which uses a callback to output information.
+class OutputStreamFormatter:
+    """Formatter which writes to an OutputStream.
 
     On instantiation, this formatter accepts a function to
     dispatch messages. Each public method accepts an object
@@ -212,23 +213,20 @@ class CallbackFormatter:
 
 
     Args:
-        callback (:obj:`callable`): A callable which can be
-            be called with a string to be output.
-        verbosity (:obj:`int`): An integer specifying how
-            verbose the output should be.
-        filter_empty (:obj:`bool`): If True, empty messages
-            will not be dispatched.
-
+        output_stream: Output is sent here
+        verbosity: Specifies how verbose output should be
+        filter_empty: If True, empty messages will not be dispatched
+        output_line_length: Maximum line length
     """
 
     def __init__(
         self,
-        callback: Callable,
+        output_stream: OutputStream,
         verbosity: int = 0,
         filter_empty: bool = True,
         output_line_length: int = 80,
     ):
-        self._callback = callback
+        self._output_stream = output_stream
         self._verbosity = verbosity
         self._filter_empty = filter_empty
         self.output_line_length = output_line_length
@@ -240,7 +238,7 @@ def _dispatch(self, s: str) -> None:
         """
         # The strip here is to filter out any empty messages
         if (not self._filter_empty) or s.strip(" \n\t"):
-            self._callback(s)
+            self._output_stream.write(s)
 
     def _format_config(self, linter: Linter) -> str:
         """Format the config of a `Linter`."""
diff --git a/src/sqlfluff/cli/outputstream.py b/src/sqlfluff/cli/outputstream.py
new file mode 100644
--- /dev/null
+++ b/src/sqlfluff/cli/outputstream.py
@@ -0,0 +1,76 @@
+"""Classes for managing linter output, used with OutputStreamFormatter."""
+import abc
+import os
+from typing import Any, Optional
+
+import click
+from tqdm import tqdm
+
+from sqlfluff.core import FluffConfig
+from sqlfluff.core.enums import FormatType
+
+
+class OutputStream(abc.ABC):
+    """Base class for linter output stream."""
+
+    def __init__(self, config: FluffConfig, context: Any = None):
+        self.config = config
+
+    def write(self, message: str) -> None:
+        """Write message to output."""
+        raise NotImplementedError  # pragma: no cover
+
+    def close(self):
+        """Close output stream."""
+        pass
+
+
+class TqdmOutput(OutputStream):
+    """Outputs to stdout, coordinates to avoid conflict with tqdm.
+
+    It may happen that progressbar conflicts with extra printing. Nothing very
+    serious happens then, except that there is printed (not removed) progressbar
+    line. The `external_write_mode` allows to disable tqdm for writing time.
+    """
+
+    def __init__(self, config: FluffConfig):
+        super().__init__(config)
+
+    def write(self, message: str) -> None:
+        """Write message to stdout."""
+        with tqdm.external_write_mode():
+            click.echo(message=message, color=self.config.get("color"))
+
+
+class FileOutput(OutputStream):
+    """Outputs to a specified file."""
+
+    def __init__(self, config: FluffConfig, output_path: str):
+        super().__init__(config)
+        self.file = open(output_path, "w")
+
+    def write(self, message: str) -> None:
+        """Write message to output_path."""
+        print(message, file=self.file)
+
+    def close(self):
+        """Close output file."""
+        self.file.close()
+
+
+def make_output_stream(
+    config: FluffConfig,
+    format: Optional[str] = None,
+    output_path: Optional[str] = None,
+) -> OutputStream:
+    """Create and return appropriate OutputStream instance."""
+    if format is None or format == FormatType.human.value:
+        if not output_path:
+            # Human-format output to stdout.
+            return TqdmOutput(config)
+        else:
+            # Human-format output to a file.
+            return FileOutput(config, output_path)
+    else:
+        # Discard human output as not required
+        return FileOutput(config, os.devnull)


Test patch:
diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py
--- a/test/cli/commands_test.py
+++ b/test/cli/commands_test.py
@@ -968,11 +968,12 @@ def test__cli__command_fail_nice_not_found(command):
 
 
 @pytest.mark.parametrize(
-    "serialize", ["yaml", "json", "github-annotation", "github-annotation-native"]
+    "serialize",
+    ["human", "yaml", "json", "github-annotation", "github-annotation-native"],
 )
 @pytest.mark.parametrize("write_file", [None, "outfile"])
 def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_path):
-    """Check the general format of JSON output for multiple files.
+    """Test the output output formats for multiple files.
 
     This tests runs both stdout checking and file checking.
     """
@@ -987,9 +988,11 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_
     )
 
     if write_file:
-        target_file = os.path.join(
-            tmp_path, write_file + (".yaml" if serialize == "yaml" else ".json")
-        )
+        ext = {
+            "human": ".txt",
+            "yaml": ".yaml",
+        }
+        target_file = os.path.join(tmp_path, write_file + ext.get(serialize, ".json"))
         cmd_args += ("--write-output", target_file)
 
     # note the file is in here twice. two files = two payloads.
@@ -1004,7 +1007,9 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_
     else:
         result_payload = result.output
 
-    if serialize == "json":
+    if serialize == "human":
+        assert len(result_payload.split("\n")) == 29 if write_file else 30
+    elif serialize == "json":
         result = json.loads(result_payload)
         assert len(result) == 2
     elif serialize == "yaml":
diff --git a/test/core/linter_test.py b/test/core/linter_test.py
--- a/test/core/linter_test.py
+++ b/test/core/linter_test.py
@@ -11,7 +11,8 @@
 from sqlfluff.core.dialects import load_raw_dialect
 from sqlfluff.core.linter import runner
 from sqlfluff.core.errors import SQLLexError, SQLBaseError, SQLLintError, SQLParseError
-from sqlfluff.cli.formatters import CallbackFormatter
+from sqlfluff.cli.formatters import OutputStreamFormatter
+from sqlfluff.cli.outputstream import make_output_stream
 from sqlfluff.core.linter import LintingResult, NoQaDirective
 import sqlfluff.core.linter as linter
 from sqlfluff.core.parser import GreedyUntil, Ref
@@ -256,8 +257,10 @@ def imap_unordered(self, *args, **kwargs):
 
         monkeypatch.setattr(runner.MultiProcessRunner, "_create_pool", _create_pool)
 
+    config = FluffConfig(overrides={"dialect": "ansi"})
+    output_stream = make_output_stream(config, None, os.devnull)
     lntr = Linter(
-        formatter=CallbackFormatter(callback=lambda m: None, verbosity=0),
+        formatter=OutputStreamFormatter(output_stream, verbosity=0),
         dialect="ansi",
     )
     result = lntr.lint_paths(