Repository: spectacles-ci/spectacles

Issue:
Failing SQL is not saved to file
After some architectural changes, failing SQL queries are no longer being logged to file. We need to reintroduce this feature. It should probably live in `cli.py`, using the value of the `sql` key in the dictionary returned by the runner.

Previously, in single-dimension mode, spectacles would create a folder per explore with a `.sql` file for each dimension query. In batch mode, it would create a `.sql` file per explore in the base directory.

These files were all created in the `logs` directory.

Comments:

Patch:
diff --git a/spectacles/cli.py b/spectacles/cli.py
index 9b7a275..15f244f 100644
--- a/spectacles/cli.py
+++ b/spectacles/cli.py
@@ -5,17 +5,14 @@ from yaml.parser import ParserError
 import argparse
 import logging
 import os
-from typing import Callable, Iterable, List
+from typing import Callable, Iterable, List, Optional
 from spectacles import __version__
 from spectacles.runner import Runner
 from spectacles.client import LookerClient
-from spectacles.exceptions import SpectaclesException, ValidationError
-from spectacles.logger import GLOBAL_LOGGER as logger, FileFormatter
+from spectacles.exceptions import SpectaclesException, ValidationError, SqlError
+from spectacles.logger import GLOBAL_LOGGER as logger, set_file_handler
 import spectacles.printer as printer
 
-LOG_FILENAME = "spectacles.log"
-LOG_FILEPATH = Path()
-
 
 class ConfigFileAction(argparse.Action):
     """Parses an arbitrary config file and assigns its values as arg defaults."""
@@ -142,7 +139,7 @@ def handle_exceptions(function: Callable) -> Callable:
             logger.debug(error, exc_info=True)
             logger.error(
                 f'Encountered unexpected {error.__class__.__name__}: "{error}"\n'
-                f"Full error traceback logged to {LOG_FILEPATH}\n\n"
+                f"Full error traceback logged to file.\n\n"
                 + printer.dim(
                     "For support, please create an issue at "
                     "https://github.com/spectacles-ci/spectacles/issues"
@@ -154,23 +151,6 @@ def handle_exceptions(function: Callable) -> Callable:
     return wrapper
 
 
-def set_file_handler(directory: str) -> None:
-
-    global LOG_FILEPATH
-
-    log_directory = Path(directory)
-    LOG_FILEPATH = Path(log_directory / LOG_FILENAME)
-    log_directory.mkdir(exist_ok=True)
-
-    fh = logging.FileHandler(LOG_FILEPATH)
-    fh.setLevel(logging.DEBUG)
-
-    formatter = FileFormatter("%(asctime)s %(levelname)s | %(message)s")
-    fh.setFormatter(formatter)
-
-    logger.addHandler(fh)
-
-
 @handle_exceptions
 def main():
     """Runs main function. This is the entry point."""
@@ -191,6 +171,7 @@ def main():
         )
     elif args.command == "sql":
         run_sql(
+            args.log_dir,
             args.project,
             args.branch,
             args.explores,
@@ -440,6 +421,31 @@ def _build_assert_subparser(
     )
 
 
+def log_failing_sql(
+    error: SqlError,
+    log_dir: str,
+    model_name: str,
+    explore_name: str,
+    dimension_name: Optional[str] = None,
+):
+
+    file_name = (
+        model_name
+        + "__"
+        + explore_name
+        + ("__" + dimension_name if dimension_name else "")
+        + ".sql"
+    )
+    file_path = Path(log_dir) / "queries" / file_name
+    print(file_path)
+
+    logger.debug(f"Logging failing SQL query for '{error.path}' to '{file_path}'")
+    logger.debug(f"Failing SQL for {error.path}: \n{error.sql}")
+
+    with open(file_path, "w") as file:
+        file.write(error.sql)
+
+
 def run_connect(
     base_url: str, client_id: str, client_secret: str, port: int, api_version: float
 ) -> None:
@@ -471,6 +477,7 @@ def run_assert(
 
 
 def run_sql(
+    log_dir,
     project,
     branch,
     explores,
@@ -508,9 +515,17 @@ def run_sql(
             for explore in iter_errors(model.explores):
                 if explore.error:
                     printer.print_sql_error(explore.error)
+                    log_failing_sql(explore.error, log_dir, model.name, explore.name)
                 else:
                     for dimension in iter_errors(explore.dimensions):
                         printer.print_sql_error(dimension.error)
+                        log_failing_sql(
+                            dimension.error,
+                            log_dir,
+                            model.name,
+                            explore.name,
+                            dimension.name,
+                        )
 
         logger.info("")
         raise ValidationError
diff --git a/spectacles/logger.py b/spectacles/logger.py
index 2c3696f..844fd49 100644
--- a/spectacles/logger.py
+++ b/spectacles/logger.py
@@ -1,7 +1,8 @@
+from pathlib import Path
 import logging
 import colorama  # type: ignore
 
-
+LOG_FILENAME = "spectacles.log"
 COLORS = {
     "red": colorama.Fore.RED,
     "green": colorama.Fore.GREEN,
@@ -12,6 +13,33 @@ COLORS = {
     "reset": colorama.Style.RESET_ALL,
 }
 
+logger = logging.getLogger("spectacles")
+logger.setLevel(logging.DEBUG)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+
+logger.addHandler(ch)
+
+GLOBAL_LOGGER = logger
+
+
+def set_file_handler(log_dir: str) -> None:
+    log_dir_path = Path(log_dir)
+    LOG_FILEPATH = log_dir_path / LOG_FILENAME
+    log_dir_path.mkdir(exist_ok=True)
+
+    # Create subfolder to save the SQL for failed queries
+    (log_dir_path / "queries").mkdir(exist_ok=True)
+
+    fh = logging.FileHandler(LOG_FILEPATH)
+    fh.setLevel(logging.DEBUG)
+
+    formatter = FileFormatter("%(asctime)s %(levelname)s | %(message)s")
+    fh.setFormatter(formatter)
+
+    logger.addHandler(fh)
+
 
 def delete_color_codes(text: str) -> str:
     for escape_sequence in COLORS.values():
@@ -24,14 +52,3 @@ class FileFormatter(logging.Formatter):
         message = super().format(record=record)
         formatted = delete_color_codes(message)
         return formatted
-
-
-logger = logging.getLogger("spectacles")
-logger.setLevel(logging.DEBUG)
-
-ch = logging.StreamHandler()
-ch.setLevel(logging.INFO)
-
-logger.addHandler(ch)
-
-GLOBAL_LOGGER = logger


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 76dc2cc..60380eb 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,8 +1,9 @@
 from unittest.mock import patch
 import pytest
 from constants import ENV_VARS
-from spectacles.cli import main, create_parser, handle_exceptions
-from spectacles.exceptions import SpectaclesException, ValidationError
+from pathlib import Path
+from spectacles.cli import main, create_parser, handle_exceptions, log_failing_sql
+from spectacles.exceptions import SpectaclesException, ValidationError, SqlError
 
 
 @pytest.fixture
@@ -181,3 +182,51 @@ def test_bad_config_file_parameter(mock_parse_config, clean_env, parser):
 def test_parse_remote_reset_with_assert(env, parser):
     args = parser.parse_args(["assert", "--remote-reset"])
     assert args.remote_reset
+
+
+def test_logging_failing_explore_sql(tmpdir):
+    error = SqlError(
+        path="example_explore",
+        message="example error message",
+        sql="select example_explore.example_dimension_1 from model",
+        explore_url="https://example.looker.com/x/12345",
+    )
+
+    query_directory = Path(tmpdir / "queries")
+    query_directory.mkdir(exist_ok=True)
+    query_file = Path(query_directory / "explore_model__example_explore.sql")
+
+    log_failing_sql(error, tmpdir, "explore_model", "example_explore")
+    content = open(query_file).read()
+
+    assert Path.exists(query_file)
+    assert content == "select example_explore.example_dimension_1 from model"
+
+
+def test_logging_failing_dimension_sql(tmpdir):
+    error = SqlError(
+        path="example_explore",
+        message="example error message",
+        sql="select example_explore.example_dimension_1 from model",
+        explore_url="https://example.looker.com/x/12345",
+    )
+
+    query_directory = Path(tmpdir / "queries")
+    query_directory.mkdir(exist_ok=True)
+    query_file = (
+        query_directory
+        / "explore_model__example_explore__example_explore.example_dimension_1.sql"
+    )
+
+    log_failing_sql(
+        error,
+        tmpdir,
+        "explore_model",
+        "example_explore",
+        "example_explore.example_dimension_1",
+    )
+
+    content = open(query_file).read()
+
+    assert content == "select example_explore.example_dimension_1 from model"
+    assert Path.exists(query_file)