Repository: sqlfluff/sqlfluff

Issue:
Suppress dbt logs and warnings when using --format github-annotation
Sometimes, running:
```
sqlfluff lint --format github-annotation --annotation-level failure --nofail 
```

Can result in the first couple of output lines being logs which break the annotations, for example:
```
14:21:42  Partial parse save file not found. Starting full parse.
Warning:  [WARNING]: Did not find matching node for patch with name 'xxxx' in the 'models' section of file 'models/production/xxxxx/xxxxx.yml'
```

## Version
dbt 1.0.0, SQLFLuff 0.9.0



Discussion:
my workaround was to add `sed -i '/^\[/!d' annotations.json` to the git actions command to delete the extra lines from dbt that were not part of the annotations beginning with `[`
Perhaps the better solution here is to add an ability for SQLFluff to write an annotations.json file itself with a command like
```
sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} --write-output annotations.json
```
which would still allow the user to see log outputs, rather than the user having to stream the logs into a file with:

```
sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
```
Relates to https://github.com/sqlfluff/sqlfluff-github-actions/issues/15
@NiallRees: That sounds like a great suggestion -- I had the same thought. I was the original author of the `github-annotation` format, and it seemed natural to add it to the existing list of formats. TBH, of the 4 formats, only one is intended for humans. If we make this change, I suggest we consider changing all the formats to support this.

Patch:
diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py
--- a/src/sqlfluff/cli/commands.py
+++ b/src/sqlfluff/cli/commands.py
@@ -6,13 +6,7 @@
 import logging
 import time
 from logging import LogRecord
-from typing import (
-    Callable,
-    Tuple,
-    NoReturn,
-    Optional,
-    List,
-)
+from typing import Callable, Tuple, NoReturn, Optional, List, cast
 
 import yaml
 
@@ -418,6 +412,17 @@ def dialects(**kwargs) -> None:
     click.echo(format_dialects(dialect_readout), color=c.get("color"))
 
 
+def dump_file_payload(filename: Optional[str], payload: str):
+    """Write the output file content to stdout or file."""
+    # If there's a file specified to write to, write to it.
+    if filename:
+        with open(filename, "w") as out_file:
+            out_file.write(payload)
+    # Otherwise write to stdout
+    else:
+        click.echo(payload)
+
+
 @cli.command()
 @common_options
 @core_options
@@ -429,6 +434,14 @@ def dialects(**kwargs) -> None:
     type=click.Choice([ft.value for ft in FormatType], case_sensitive=False),
     help="What format to return the lint result in (default=human).",
 )
+@click.option(
+    "--write-output",
+    help=(
+        "Optionally provide a filename to write the results to, mostly used in "
+        "tandem with --format. NB: Setting an output file re-enables normal "
+        "stdout logging."
+    ),
+)
 @click.option(
     "--annotation-level",
     default="notice",
@@ -468,6 +481,7 @@ def lint(
     paths: Tuple[str],
     processes: int,
     format: str,
+    write_output: Optional[str],
     annotation_level: str,
     nofail: bool,
     disregard_sqlfluffignores: bool,
@@ -497,7 +511,8 @@ def lint(
 
     """
     config = get_config(extra_config_path, ignore_local_config, **kwargs)
-    non_human_output = format != FormatType.human.value
+    non_human_output = (format != FormatType.human.value) or (write_output is not None)
+    file_output = None
     lnt, formatter = get_linter_and_formatter(config, silent=non_human_output)
 
     verbose = config.get("verbose")
@@ -535,9 +550,9 @@ def lint(
             click.echo(format_linting_stats(result, verbose=verbose))
 
     if format == FormatType.json.value:
-        click.echo(json.dumps(result.as_records()))
+        file_output = json.dumps(result.as_records())
     elif format == FormatType.yaml.value:
-        click.echo(yaml.dump(result.as_records(), sort_keys=False))
+        file_output = yaml.dump(result.as_records(), sort_keys=False)
     elif format == FormatType.github_annotation.value:
         github_result = []
         for record in result.as_records():
@@ -558,7 +573,10 @@ def lint(
                         "annotation_level": annotation_level,
                     }
                 )
-        click.echo(json.dumps(github_result))
+        file_output = json.dumps(github_result)
+
+    if file_output:
+        dump_file_payload(write_output, cast(str, file_output))
 
     if bench:
         click.echo("==== overall timings ====")
@@ -876,6 +894,14 @@ def quoted_presenter(dumper, data):
     ),
     help="What format to return the parse result in.",
 )
+@click.option(
+    "--write-output",
+    help=(
+        "Optionally provide a filename to write the results to, mostly used in "
+        "tandem with --format. NB: Setting an output file re-enables normal "
+        "stdout logging."
+    ),
+)
 @click.option(
     "--profiler", is_flag=True, help="Set this flag to engage the python profiler."
 )
@@ -892,6 +918,7 @@ def parse(
     code_only: bool,
     include_meta: bool,
     format: str,
+    write_output: Optional[str],
     profiler: bool,
     bench: bool,
     nofail: bool,
@@ -909,7 +936,8 @@ def parse(
     """
     c = get_config(extra_config_path, ignore_local_config, **kwargs)
     # We don't want anything else to be logged if we want json or yaml output
-    non_human_output = format in (FormatType.json.value, FormatType.yaml.value)
+    # unless we're writing to a file.
+    non_human_output = (format != FormatType.human.value) or (write_output is not None)
     lnt, formatter = get_linter_and_formatter(c, silent=non_human_output)
     verbose = c.get("verbose")
     recurse = c.get("recurse")
@@ -975,9 +1003,12 @@ def parse(
                 # For yaml dumping always dump double quoted strings if they contain
                 # tabs or newlines.
                 yaml.add_representer(str, quoted_presenter)
-                click.echo(yaml.dump(parsed_strings_dict, sort_keys=False))
+                file_output = yaml.dump(parsed_strings_dict, sort_keys=False)
             elif format == FormatType.json.value:
-                click.echo(json.dumps(parsed_strings_dict))
+                file_output = json.dumps(parsed_strings_dict)
+
+            # Dump the output to stdout or to file as appropriate.
+            dump_file_payload(write_output, file_output)
 
     except OSError:  # pragma: no cover
         click.echo(


Test patch:
diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py
--- a/test/cli/commands_test.py
+++ b/test/cli/commands_test.py
@@ -798,19 +798,35 @@ def test__cli__command__fix_no_force(rule, fname, prompt, exit_code, fix_exit_co
 
 
 @pytest.mark.parametrize("serialize", ["yaml", "json"])
-def test__cli__command_parse_serialize_from_stdin(serialize):
+@pytest.mark.parametrize("write_file", [None, "outfile"])
+def test__cli__command_parse_serialize_from_stdin(serialize, write_file, tmp_path):
     """Check that the parser serialized output option is working.
 
+    This tests both output to stdout and output to file.
+
     Not going to test for the content of the output as that is subject to change.
     """
+    cmd_args = ("-", "--format", serialize)
+
+    if write_file:
+        target_file = os.path.join(tmp_path, write_file + "." + serialize)
+        cmd_args += ("--write-output", target_file)
+
     result = invoke_assert_code(
-        args=[parse, ("-", "--format", serialize)],
+        args=[parse, cmd_args],
         cli_input="select * from tbl",
     )
+
+    if write_file:
+        with open(target_file, "r") as payload_file:
+            result_payload = payload_file.read()
+    else:
+        result_payload = result.output
+
     if serialize == "json":
-        result = json.loads(result.output)
+        result = json.loads(result_payload)
     elif serialize == "yaml":
-        result = yaml.safe_load(result.output)
+        result = yaml.safe_load(result_payload)
     else:
         raise Exception
     result = result[0]  # only one file
@@ -880,24 +896,42 @@ def test__cli__command_fail_nice_not_found(command):
 
 
 @pytest.mark.parametrize("serialize", ["yaml", "json", "github-annotation"])
-def test__cli__command_lint_serialize_multiple_files(serialize):
-    """Check the general format of JSON output for multiple files."""
+@pytest.mark.parametrize("write_file", [None, "outfile"])
+def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_path):
+    """Check the general format of JSON output for multiple files.
+
+    This tests runs both stdout checking and file checking.
+    """
     fpath = "test/fixtures/linter/indentation_errors.sql"
 
+    cmd_args = (fpath, fpath, "--format", serialize, "--disable_progress_bar")
+
+    if write_file:
+        target_file = os.path.join(
+            tmp_path, write_file + (".yaml" if serialize == "yaml" else ".json")
+        )
+        cmd_args += ("--write-output", target_file)
+
     # note the file is in here twice. two files = two payloads.
     result = invoke_assert_code(
-        args=[lint, (fpath, fpath, "--format", serialize, "--disable_progress_bar")],
+        args=[lint, cmd_args],
         ret_code=65,
     )
 
+    if write_file:
+        with open(target_file, "r") as payload_file:
+            result_payload = payload_file.read()
+    else:
+        result_payload = result.output
+
     if serialize == "json":
-        result = json.loads(result.output)
+        result = json.loads(result_payload)
         assert len(result) == 2
     elif serialize == "yaml":
-        result = yaml.safe_load(result.output)
+        result = yaml.safe_load(result_payload)
         assert len(result) == 2
     elif serialize == "github-annotation":
-        result = json.loads(result.output)
+        result = json.loads(result_payload)
         filepaths = {r["file"] for r in result}
         assert len(filepaths) == 1
     else: