Repository: great-expectations/great_expectations

Issue:
Error when running checkpoint with temp table on Redshift
**Describe the bug**
I am running a checkpoint against Redshift db with config that utilized temp tables. When invoking `context.run_checkpoint(checkpoint_name="my_cp")` , where my_cp suite has `expect_column_values_to_be_unique` expectation, I get the following error

```ProgrammingError: (psycopg2.errors.UndefinedTable) relation "ge_tmp_617b3f37" does not exist

[SQL: INSERT INTO ge_tmp_a0647ee5 (condition) SELECT CASE WHEN (personid IS NOT NULL AND personid IN (SELECT personid 
FROM ge_tmp_617b3f37 GROUP BY personid 
HAVING count(personid) > %(count_1)s)) THEN %(param_1)s ELSE %(param_2)s END AS condition 
FROM ge_tmp_617b3f37]
[parameters: {'count_1': 1, 'param_1': 1, 'param_2': 0}]
(Background on this error at: http://sqlalche.me/e/f405)
```


Error stack
```
\lib\site-packages\great_expectations\validator\validator.py in _resolve_metrics(self, execution_engine, metrics_to_resolve, metrics, runtime_configuration)
    545         """A means of accessing the Execution Engine's resolve_metrics method, where missing metric configurations are
    546         resolved"""
--> 547         return execution_engine.resolve_metrics(
    548             metrics_to_resolve, metrics, runtime_configuration
    549         )

\lib\site-packages\great_expectations\execution_engine\execution_engine.py in resolve_metrics(self, metrics_to_resolve, metrics, runtime_configuration)
    255                 )
    256             elif metric_fn_type == MetricFunctionTypes.VALUE:
--> 257                 resolved_metrics[metric_to_resolve.id] = metric_fn(
    258                     **metric_provider_kwargs
    259                 )

\lib\site-packages\great_expectations\expectations\metrics\map_metric.py in _sqlalchemy_map_condition_unexpected_count_value(cls, execution_engine, metric_domain_kwargs, metric_value_kwargs, metrics, **kwargs)
    875                 sa.select(count_case_statement).select_from(selectable),
    876             )
--> 877             execution_engine.engine.execute(inner_case_query)
    878 
    879         unexpected_count_query: sa.Select = (

\lib\site-packages\sqlalchemy\engine\base.py in execute(self, statement, *multiparams, **params)
   2242 
   2243         connection = self._contextual_connect(close_with_result=True)
-> 2244         return connection.execute(statement, *multiparams, **params)
   2245 
   2246     def scalar(self, statement, *multiparams, **params):

\lib\site-packages\sqlalchemy\engine\base.py in execute(self, object_, *multiparams, **params)
   1018             )
   1019         else:
-> 1020             return meth(self, multiparams, params)
   1021 
   1022     def _execute_function(self, func, multiparams, params):

\lib\site-packages\sqlalchemy\sql\elements.py in _execute_on_connection(self, connection, multiparams, params)
    296     def _execute_on_connection(self, connection, multiparams, params):
    297         if self.supports_execution:
--> 298             return connection._execute_clauseelement(self, multiparams, params)
    299         else:
    300             raise exc.ObjectNotExecutableError(self)

\lib\site-packages\sqlalchemy\engine\base.py in _execute_clauseelement(self, elem, multiparams, params)
   1131             )
   1132 
-> 1133         ret = self._execute_context(
   1134             dialect,
   1135             dialect.execution_ctx_cls._init_compiled,

\lib\site-packages\sqlalchemy\engine\base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1321 
   1322         except BaseException as e:
-> 1323             self._handle_dbapi_exception(
   1324                 e, statement, parameters, cursor, context
   1325             )

\lib\site-packages\sqlalchemy\engine\base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1515                 util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   1516             elif should_wrap:
-> 1517                 util.raise_(
   1518                     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   1519                 )

\lib\site-packages\sqlalchemy\util\compat.py in raise_(***failed resolving arguments***)
    176 
    177         try:
--> 178             raise exception
    179         finally:
    180             # credit to

\lib\site-packages\sqlalchemy\engine\base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1281                             break
   1282                 if not evt_handled:
-> 1283                     self.dialect.do_execute(
   1284                         cursor, statement, parameters, context
   1285                     )

\lib\site-packages\sqlalchemy\engine\default.py in do_execute(self, cursor, statement, parameters, context)
    588 
    589     def do_execute(self, cursor, statement, parameters, context=None):
--> 590         cursor.execute(statement, parameters)
    591 
    592     def do_execute_no_params(self, cursor, statement, context=None):

```

**To Reproduce**
Steps to reproduce the behavior:
1.
 Datasource config 
```
rs_datasource:
    class_name: Datasource
    execution_engine:
      class_name: SqlAlchemyExecutionEngine
      connection_string: ${connection_str}
      module_name: great_expectations.execution_engine
    module_name: great_expectations.datasource
    data_connectors:
      my_dc:
        class_name: RuntimeDataConnector
        module_name: great_expectations.datasource.data_connector
        batch_identifiers:
          - some_identifier
  ```
Checkpoint config

```name: my_cp
config_version: 1.0
module_name: great_expectations.checkpoint
class_name: Checkpoint
runtime_configuration: 
  result_format:
        result_format: COMPLETE
validations:
  - batch_request:
      batch_identifiers:
        ge2c: ge2c
      data_asset_name: my_table_name
      data_connector_name: my_dc
      datasource_name: rs_datasource
      runtime_parameters:
        query: select * from my_table_name
    expectation_suite_name: 
````

Looks having this expectation in my suite causes the issue
```
 {
      "expectation_type": "expect_column_values_to_be_unique",
      "kwargs": {
        "column": "personid"
      },
      "meta": {}
    }
```

2. run ```context.run_checkpoint(checkpoint_name="my_cp")```

**Expected behavior**
Checkpoint completes successfully

**Environment (please complete the following information):**
 - Operating System :Windows
 - Great Expectations 0.13.19

**Additional context**
* From what I can tell, the  temp table `ge_tmp_617b3f37` was created successfully but it can't be accessed by the ```INSERT INTO``` query because it was created using [one sqlalchemy engine session](
https://github.com/great-expectations/great_expectations/blob/develop/great_expectations/execution_engine/sqlalchemy_batch_data.py#L249), while the ```INSERT INTO``` query runs from a [different sqlachemy session](
https://github.com/great-expectations/great_expectations/blob/develop/great_expectations/expectations/metrics/map_metric.py#L853). And according to [Redshift docs](https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html)
```
TEMPORARY | TEMP - Keyword that creates a temporary table that is visible only within the current session. ... 
The temporary table is created in a separate, session-specific schema. (You can't specify a name for this schema.) 
```





Discussion:

Resolving patch:
diff --git a/great_expectations/expectations/metrics/map_metric.py b/great_expectations/expectations/metrics/map_metric.py
--- a/great_expectations/expectations/metrics/map_metric.py
+++ b/great_expectations/expectations/metrics/map_metric.py
@@ -849,37 +849,45 @@ def _sqlalchemy_map_condition_unexpected_count_value(
     (selectable, _, _,) = execution_engine.get_compute_domain(
         compute_domain_kwargs, domain_type=MetricDomainTypes.IDENTITY.value
     )
-    temp_table_name: str = f"ge_tmp_{str(uuid.uuid4())[:8]}"
-    if execution_engine.dialect == "mssql":
-        # mssql expects all temporary table names to have a prefix '#'
-        temp_table_name = f"#{temp_table_name}"
+
+    count_case_statement: List[sa.sql.elements.Label] = [
+        sa.case(
+            [
+                (
+                    unexpected_condition,
+                    1,
+                )
+            ],
+            else_=0,
+        ).label("condition")
+    ]
 
     try:
-        with execution_engine.engine.begin():
-            metadata: sa.MetaData = sa.MetaData(execution_engine.engine)
-            temp_table_obj: sa.Table = sa.Table(
-                temp_table_name,
-                metadata,
-                sa.Column("condition", sa.Integer, primary_key=False, nullable=False),
-            )
-            temp_table_obj.create(execution_engine.engine, checkfirst=True)
-
-            count_case_statement: List[sa.sql.elements.Label] = [
-                sa.case(
-                    [
-                        (
-                            unexpected_condition,
-                            1,
-                        )
-                    ],
-                    else_=0,
-                ).label("condition")
-            ]
-            inner_case_query: sa.sql.dml.Insert = temp_table_obj.insert().from_select(
-                count_case_statement,
-                sa.select(count_case_statement).select_from(selectable),
-            )
-            execution_engine.engine.execute(inner_case_query)
+        if execution_engine.engine.dialect.name.lower() == "mssql":
+            temp_table_name: str = f"#ge_tmp_{str(uuid.uuid4())[:8]}"
+
+            with execution_engine.engine.begin():
+                metadata: sa.MetaData = sa.MetaData(execution_engine.engine)
+                temp_table_obj: sa.Table = sa.Table(
+                    temp_table_name,
+                    metadata,
+                    sa.Column(
+                        "condition", sa.Integer, primary_key=False, nullable=False
+                    ),
+                )
+                temp_table_obj.create(execution_engine.engine, checkfirst=True)
+
+                inner_case_query: sa.sql.dml.Insert = (
+                    temp_table_obj.insert().from_select(
+                        count_case_statement,
+                        sa.select(count_case_statement).select_from(selectable),
+                    )
+                )
+                execution_engine.engine.execute(inner_case_query)
+
+                selectable_count = temp_table_obj
+        else:
+            selectable_count = sa.select(count_case_statement).select_from(selectable)
 
         unexpected_count_query: sa.Select = (
             sa.select(
@@ -887,7 +895,7 @@ def _sqlalchemy_map_condition_unexpected_count_value(
                     sa.func.sum(sa.column("condition")).label("unexpected_count"),
                 ]
             )
-            .select_from(temp_table_obj)
+            .select_from(selectable_count)
             .alias("UnexpectedCountSubquery")
         )
 


Test patch:
diff --git a/tests/execution_engine/test_sqlalchemy_execution_engine.py b/tests/execution_engine/test_sqlalchemy_execution_engine.py
--- a/tests/execution_engine/test_sqlalchemy_execution_engine.py
+++ b/tests/execution_engine/test_sqlalchemy_execution_engine.py
@@ -21,7 +21,7 @@
 from great_expectations.self_check.util import build_sa_engine
 from great_expectations.validator.validation_graph import MetricConfiguration
 from tests.expectations.test_util import get_table_columns_metric
-from tests.test_utils import get_sqlite_temp_table_names
+from tests.test_utils import get_sqlite_table_names, get_sqlite_temp_table_names
 
 try:
     sqlalchemy = pytest.importorskip("sqlalchemy")
@@ -562,3 +562,62 @@ def test_get_batch_data_and_markers_using_query(sqlite_view_engine, test_df):
 
     assert len(get_sqlite_temp_table_names(sqlite_view_engine)) == 2
     assert batch_markers.get("ge_load_time") is not None
+
+
+def test_sa_batch_unexpected_condition_temp_table(caplog, sa):
+    def validate_tmp_tables():
+        temp_tables = [
+            name
+            for name in get_sqlite_temp_table_names(engine.engine)
+            if name.startswith("ge_tmp_")
+        ]
+        tables = [
+            name
+            for name in get_sqlite_table_names(engine.engine)
+            if name.startswith("ge_tmp_")
+        ]
+        assert len(temp_tables) == 0
+        assert len(tables) == 0
+
+    engine = build_sa_engine(
+        pd.DataFrame({"a": [1, 2, 1, 2, 3, 3], "b": [4, 4, 4, 4, 4, 4]}), sa
+    )
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    validate_tmp_tables()
+
+    condition_metric = MetricConfiguration(
+        metric_name="column_values.unique.condition",
+        metric_domain_kwargs={"column": "a"},
+        metric_value_kwargs=dict(),
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(condition_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    validate_tmp_tables()
+
+    desired_metric = MetricConfiguration(
+        metric_name="column_values.unique.unexpected_count",
+        metric_domain_kwargs={"column": "a"},
+        metric_value_kwargs=dict(),
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(desired_metric,), metrics=metrics
+    )
+
+    validate_tmp_tables()
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -178,6 +178,19 @@ def get_sqlite_temp_table_names(engine):
     return {row[0] for row in rows}
 
 
+def get_sqlite_table_names(engine):
+    result = engine.execute(
+        """
+SELECT
+    name
+FROM
+    sqlite_master
+"""
+    )
+    rows = result.fetchall()
+    return {row[0] for row in rows}
+
+
 def build_in_memory_store_backend(
     module_name: str = "great_expectations.data_context.store",
     class_name: str = "InMemoryStoreBackend",