Repository: dbt-labs/dbt-core

Issue:
[Bug] Support tests on databases that lack real boolean types
We define `TestResultData` as a tripartite collection of one integer and two boolean values:

https://github.com/dbt-labs/dbt-core/blob/26625e9627261f12a278828a43defe750942cb4f/core/dbt/task/test.py#L31-L35

And we validate it here, to ensure that the test materialization's `main` query has indeed returned an integer and two booleans:

https://github.com/dbt-labs/dbt-core/blob/26625e9627261f12a278828a43defe750942cb4f/core/dbt/task/test.py#L106-L107

This is quite tricky (perhaps impossible) for databases that don't support a real boolean type. (See: https://github.com/dbt-msft/dbt-sqlserver/issues/164.) In past versions (v0.20), dbt would use agate to coerce data types in query results (e.g. `'true'` &rarr; `True`), but we don't do that anymore.

I think we could take one of two approaches:
1. Use agate to introspect query results, and see if the database-returned type maps to boolean, using (e.g.) the [`convert_boolean_type`](https://github.com/dbt-msft/dbt-sqlserver/blob/4091a07a5bdb7b5bac521fc72d8da52fabcd6c16/dbt/adapters/sqlserver/impl.py#L31-L33) method that we ask all adapter plugins to implement for agate/seed purposes
2. Use python's [built-in `bool()` function](https://docs.python.org/3/library/functions.html#bool), to [truth-test](https://docs.python.org/3/library/stdtypes.html#truth) string/integer types returned by the database. This works well with databases (e.g. SQLServer) that represent boolean values as `0` or `1`

I lean toward the latter, for simplicity, as long as truth-testing database-returned values doesn't pose any risk. (I think it's safer/better than `eval()`?) Implementing this could be as simple as adding a custom `validate` method to `TestResultData`:

```python
@dataclass
class TestResultData(dbtClassMixin):
    failures: int
    should_warn: bool
    should_error: bool
    
    @classmethod
    def validate(cls, data):
        data['should_warn'] = bool(data.get('should_warn'))
        data['should_error'] = bool(data.get('should_error'))
        super().validate(data)
```

This works for me in local testing, both with the [default version of `get_test_sql`](https://github.com/dbt-labs/dbt-core/blob/26625e9627261f12a278828a43defe750942cb4f/core/dbt/include/global_project/macros/materializations/test.sql#L6), and if I override it locally to be:
```sql
{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
    select
      {{ fail_calc }} as failures,
      case when {{ fail_calc }} {{ warn_if }} then 1 else 0 end as should_warn,
      case when {{ fail_calc }} {{ error_if }} then 1 else 0 end as should_error
    from (
      {{ main_sql }}
      {{ "limit " ~ limit if limit != none }}
    ) dbt_internal_test
{%- endmacro %}
```


Discussion:

Resolving patch:
diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py
--- a/core/dbt/exceptions.py
+++ b/core/dbt/exceptions.py
@@ -466,6 +466,15 @@ def invalid_type_error(method_name, arg_name, got_value, expected_type,
                          got_value=got_value, got_type=got_type))
 
 
+def invalid_bool_error(got_value, macro_name) -> NoReturn:
+    """Raise a CompilationException when an macro expects a boolean but gets some
+    other value.
+    """
+    msg = ("Macro '{macro_name}' returns '{got_value}'.  It is not type 'bool' "
+           "and cannot not be converted reliably to a bool.")
+    raise_compiler_error(msg.format(macro_name=macro_name, got_value=got_value))
+
+
 def ref_invalid_args(model, args) -> NoReturn:
     raise_compiler_error(
         "ref() takes at most two arguments ({} given)".format(len(args)),


Test patch:
diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py
--- a/core/dbt/task/test.py
+++ b/core/dbt/task/test.py
@@ -1,3 +1,5 @@
+from distutils.util import strtobool
+
 from dataclasses import dataclass
 from dbt import utils
 from dbt.dataclass_schema import dbtClassMixin
@@ -19,6 +21,7 @@
 from dbt.clients.jinja import MacroGenerator
 from dbt.exceptions import (
     InternalException,
+    invalid_bool_error,
     missing_materialization
 )
 from dbt.graph import (
@@ -34,6 +37,23 @@ class TestResultData(dbtClassMixin):
     should_warn: bool
     should_error: bool
 
+    @classmethod
+    def validate(cls, data):
+        data['should_warn'] = cls.convert_bool_type(data['should_warn'])
+        data['should_error'] = cls.convert_bool_type(data['should_error'])
+        super().validate(data)
+
+    def convert_bool_type(field) -> bool:
+        # if it's type string let python decide if it's a valid value to convert to bool
+        if isinstance(field, str):
+            try:
+                return bool(strtobool(field))  # type: ignore
+            except ValueError:
+                raise invalid_bool_error(field, 'get_test_sql')
+
+        # need this so we catch both true bools and 0/1
+        return bool(field)
+
 
 class TestRunner(CompileRunner):
     def describe_node(self):
diff --git a/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros/get_test_sql.sql b/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros/get_test_sql.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros/get_test_sql.sql
@@ -0,0 +1,10 @@
+{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+    select
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }} then 1 else 0 end as should_warn,
+      case when {{ fail_calc }} {{ error_if }} then 1 else 0 end as should_error
+    from (
+      {{ main_sql }}
+      {{ "limit " ~ limit if limit != none }}
+    ) dbt_internal_test
+{%- endmacro %}
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros_fail/get_test_sql.sql b/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros_fail/get_test_sql.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/macros-v2/override_get_test_macros_fail/get_test_sql.sql
@@ -0,0 +1,10 @@
+{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+    select
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }} then 'x' else 'y' end as should_warn,
+      case when {{ fail_calc }} {{ error_if }} then 'x' else 'y' end as should_error
+    from (
+      {{ main_sql }}
+      {{ "limit " ~ limit if limit != none }}
+    ) dbt_internal_test
+{% endmacro %}
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_failure.sql b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_failure.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_failure.sql
@@ -0,0 +1,3 @@
+select * from {{ ref('my_model_pass') }}
+UNION ALL
+select null as id
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_pass.sql b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_pass.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_pass.sql
@@ -0,0 +1,3 @@
+select 1 as id
+UNION ALL
+select null as id
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_warning.sql b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_warning.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/my_model_warning.sql
@@ -0,0 +1 @@
+select * from {{ ref('my_model_pass') }}
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models/schema.yml b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models/schema.yml
@@ -0,0 +1,31 @@
+version: 2
+
+models:
+    - name: my_model_pass
+      description: "The table has 1 null values, and we're okay with that, until it's more than 1."
+      columns:
+        - name: id
+          description: "The number of responses for this favorite color - purple will be null"
+          tests:
+            - not_null:
+                error_if: '>1'
+                warn_if: '>1'
+
+    - name: my_model_warning
+      description: "The table has 1 null values, and we're okay with that, but let us know"
+      columns:
+        - name: id
+          description: "The number of responses for this favorite color - purple will be null"
+          tests:
+            - not_null:
+                error_if: '>1'
+
+    - name: my_model_failure
+      description: "The table has 2 null values, and we're not okay with that"
+      columns:
+        - name: id
+          description: "The number of responses for this favorite color - purple will be null"
+          tests:
+            - not_null:
+                error_if: '>1'
+
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/my_model.sql b/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/my_model.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/my_model.sql
@@ -0,0 +1,3 @@
+select 1 as id
+UNION ALL
+select null as id
\ No newline at end of file
diff --git a/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/schema.yml b/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/schema.yml
new file mode 100644
--- /dev/null
+++ b/test/integration/008_schema_tests_test/models-v2/override_get_test_models_fail/schema.yml
@@ -0,0 +1,12 @@
+version: 2
+
+models:
+    - name: my_model
+      description: "The table has 1 null values, and we're not okay with that."
+      columns:
+        - name: id
+          description: "The number of responses for this favorite color - purple will be null"
+          tests:
+            - not_null
+
+
diff --git a/test/integration/008_schema_tests_test/test_schema_v2_tests.py b/test/integration/008_schema_tests_test/test_schema_v2_tests.py
--- a/test/integration/008_schema_tests_test/test_schema_v2_tests.py
+++ b/test/integration/008_schema_tests_test/test_schema_v2_tests.py
@@ -159,6 +159,178 @@ def test_postgres_limit_schema_tests(self):
         self.assertEqual(sum(x.failures for x in test_results), 3)
 
 
+class TestDefaultBoolType(DBTIntegrationTest):
+    # test with default True/False in get_test_sql macro
+
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+
+    @property
+    def schema(self):
+        return "schema_tests_008"
+
+    @property
+    def models(self):
+        return "models-v2/override_get_test_models"
+
+    def run_schema_validations(self):
+        args = FakeArgs()
+        test_task = TestTask(args, self.config)
+        return test_task.run()
+
+    def assertTestFailed(self, result):
+        self.assertEqual(result.status, "fail")
+        self.assertFalse(result.skipped)
+        self.assertTrue(
+            result.failures > 0,
+            'test {} did not fail'.format(result.node.name)
+        )
+
+    def assertTestWarn(self, result):
+        self.assertEqual(result.status, "warn")
+        self.assertFalse(result.skipped)
+        self.assertTrue(
+            result.failures > 0,
+            'test {} passed without expected warning'.format(result.node.name)
+        )
+
+    def assertTestPassed(self, result):
+        self.assertEqual(result.status, "pass")
+        self.assertFalse(result.skipped)
+        self.assertEqual(
+            result.failures, 0,
+            'test {} failed'.format(result.node.name)
+        )
+
+    @use_profile('postgres')
+    def test_postgres_limit_schema_tests(self):
+        results = self.run_dbt()
+        self.assertEqual(len(results), 3)
+        test_results = self.run_schema_validations()
+        self.assertEqual(len(test_results), 3)
+
+        for result in test_results:
+            # assert that all deliberately failing tests actually fail
+            if 'failure' in result.node.name:
+                self.assertTestFailed(result)
+            # assert that tests with warnings have them
+            elif 'warning' in result.node.name:
+                self.assertTestWarn(result)
+            # assert that actual tests pass
+            else:
+                self.assertTestPassed(result)
+        # warnings are also marked as failures
+        self.assertEqual(sum(x.failures for x in test_results), 3)
+
+
+class TestOtherBoolType(DBTIntegrationTest):
+    # test with expected 0/1 in custom get_test_sql macro
+
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+
+    @property
+    def schema(self):
+        return "schema_tests_008"
+
+    @property
+    def models(self):
+        return "models-v2/override_get_test_models"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            "macro-paths": ["macros-v2/override_get_test_macros"],
+        }
+
+    def run_schema_validations(self):
+        args = FakeArgs()
+        test_task = TestTask(args, self.config)
+        return test_task.run()
+
+    def assertTestFailed(self, result):
+        self.assertEqual(result.status, "fail")
+        self.assertFalse(result.skipped)
+        self.assertTrue(
+            result.failures > 0,
+            'test {} did not fail'.format(result.node.name)
+        )
+
+    def assertTestWarn(self, result):
+        self.assertEqual(result.status, "warn")
+        self.assertFalse(result.skipped)
+        self.assertTrue(
+            result.failures > 0,
+            'test {} passed without expected warning'.format(result.node.name)
+        )
+
+    def assertTestPassed(self, result):
+        self.assertEqual(result.status, "pass")
+        self.assertFalse(result.skipped)
+        self.assertEqual(
+            result.failures, 0,
+            'test {} failed'.format(result.node.name)
+        )
+
+    @use_profile('postgres')
+    def test_postgres_limit_schema_tests(self):
+        results = self.run_dbt()
+        self.assertEqual(len(results), 3)
+        test_results = self.run_schema_validations()
+        self.assertEqual(len(test_results), 3)
+
+        for result in test_results:
+            # assert that all deliberately failing tests actually fail
+            if 'failure' in result.node.name:
+                self.assertTestFailed(result)
+            # assert that tests with warnings have them
+            elif 'warning' in result.node.name:
+                self.assertTestWarn(result)
+            # assert that actual tests pass
+            else:
+                self.assertTestPassed(result)
+        # warnings are also marked as failures
+        self.assertEqual(sum(x.failures for x in test_results), 3)
+
+
+class TestNonBoolType(DBTIntegrationTest):
+    # test with invalid 'x'/'y' in custom get_test_sql macro
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+
+    @property
+    def schema(self):
+        return "schema_tests_008"
+
+    @property
+    def models(self):
+        return "models-v2/override_get_test_models_fail"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            "macro-paths": ["macros-v2/override_get_test_macros_fail"],
+        }
+
+    def run_schema_validations(self):
+        args = FakeArgs()
+
+        test_task = TestTask(args, self.config)
+        return test_task.run()
+
+    @use_profile('postgres')
+    def test_postgres_limit_schema_tests(self):
+        results = self.run_dbt()
+        self.assertEqual(len(results), 1)
+        run_result = self.run_dbt(['test'], expect_pass=False)
+        results = run_result.results
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].status, TestStatus.Error)
+        self.assertRegex(results[0].message, r"'get_test_sql' returns 'x'")
+
+
 class TestMalformedSchemaTests(DBTIntegrationTest):
 
     def setUp(self):
@@ -621,4 +793,3 @@ def test_postgres_wrong_specification_block(self):
 
         assert len(results) == 1
         assert results[0] == '{"name": "some_seed", "description": ""}'
-