Repository: great-expectations/great_expectations

Issue:
custom_sql fails for Redshift databases
Redshift unfortunately does not have functionality for `IF NOT EXISTS`, so the temporary table creation fails when passing custom sql to a Redshift SQLAlchemy engine. It seems like the options are here are to introspect the engine and special-case Redshift (seems harder to maintain), to remove the `IF NOT EXISTS` for everyone and let the custom sql fail if the table exists (I think I'd expect this behavior as a user anyway), or to not support custom_sql for Redshift.


Discussion:
If the maintainers have thoughts on which approach makes the most sense for the library, I'm happy to take a stab at this next week as well. I'm hoping to make some contributions to this library for Hacktoberfest :)
Cool! I haven't looked into it but the second option makes sense to me. @jpcampb2 is the author and resident expert on the SQLAlchemy implementations. 

Resolving patch:
diff --git a/great_expectations/dataset/sqlalchemy_dataset.py b/great_expectations/dataset/sqlalchemy_dataset.py
--- a/great_expectations/dataset/sqlalchemy_dataset.py
+++ b/great_expectations/dataset/sqlalchemy_dataset.py
@@ -86,16 +86,20 @@ def inner_wrapper(self, column, mostly=None, result_format=None, *args, **kwargs
                             sa.or_(
                                 # SA normally evaluates `== None` as `IS NONE`. However `sa.in_()`
                                 # replaces `None` as `NULL` in the list and incorrectly uses `== NULL`
-                                sa.column(column).is_(None) == False if None in ignore_values else False,
+                                sa.column(column).is_(
+                                    None) == False if None in ignore_values else False,
                                 # Ignore any other values that are in the ignore list
                                 sa.column(column).in_(ignore_values) == False))
                 ).limit(unexpected_count_limit)
             )
 
-            nonnull_count = count_results['element_count'] - count_results['null_count']
-            maybe_limited_unexpected_list = [x[column] for x in unexpected_query_results.fetchall()]
+            nonnull_count = count_results['element_count'] - \
+                count_results['null_count']
+            maybe_limited_unexpected_list = [x[column]
+                                             for x in unexpected_query_results.fetchall()]
             success_count = nonnull_count - count_results['unexpected_count']
-            success, percent_success = self._calc_map_expectation_success(success_count, nonnull_count, mostly)
+            success, percent_success = self._calc_map_expectation_success(
+                success_count, nonnull_count, mostly)
 
             return_obj = self._format_column_map_output(
                 result_format, success,
@@ -118,7 +122,6 @@ def inner_wrapper(self, column, mostly=None, result_format=None, *args, **kwargs
 
         return inner_wrapper
 
-
     @classmethod
     def column_aggregate_expectation(cls, func):
         """Constructs an expectation using column-aggregate semantics.
@@ -130,7 +133,7 @@ def column_aggregate_expectation(cls, func):
 
         @cls.expectation(argspec)
         @wraps(func)
-        def inner_wrapper(self, column, result_format = None, *args, **kwargs):
+        def inner_wrapper(self, column, result_format=None, *args, **kwargs):
 
             if result_format is None:
                 result_format = self.default_expectation_args["result_format"]
@@ -140,13 +143,16 @@ def inner_wrapper(self, column, result_format = None, *args, **kwargs):
             evaluation_result = func(self, column, *args, **kwargs)
 
             if 'success' not in evaluation_result:
-                raise ValueError("Column aggregate expectation failed to return required information: success")
+                raise ValueError(
+                    "Column aggregate expectation failed to return required information: success")
 
             if 'result' not in evaluation_result:
-                raise ValueError("Column aggregate expectation failed to return required information: result")
+                raise ValueError(
+                    "Column aggregate expectation failed to return required information: result")
 
             if 'observed_value' not in evaluation_result['result']:
-                raise ValueError("Column aggregate expectation failed to return required information: result.observed_value")
+                raise ValueError(
+                    "Column aggregate expectation failed to return required information: result.observed_value")
 
             return_obj = {
                 'success': bool(evaluation_result['success'])
@@ -166,7 +172,8 @@ def inner_wrapper(self, column, result_format = None, *args, **kwargs):
                     ).label('null_count'),
                 ]).select_from(self._table)
 
-                count_results = dict(self.engine.execute(count_query).fetchone())
+                count_results = dict(
+                    self.engine.execute(count_query).fetchone())
 
                 # Handle case of empty table gracefully:
                 if "element_count" not in count_results or count_results["element_count"] is None:
@@ -197,7 +204,8 @@ def inner_wrapper(self, column, result_format = None, *args, **kwargs):
             if result_format['result_format'] in ["SUMMARY", "COMPLETE"]:
                 return return_obj
 
-            raise ValueError("Unknown result_format %s." % (result_format['result_format'],))
+            raise ValueError("Unknown result_format %s." %
+                             (result_format['result_format'],))
 
         return inner_wrapper
 
@@ -247,17 +255,18 @@ def create_temporary_table(self, table_name, custom_sql):
         It hasn't been tested in all SQL dialects, and may change based on community feedback.
         :param custom_sql:
         """
-        stmt = "CREATE TEMPORARY TABLE IF NOT EXISTS {table_name} AS {custom_sql}".format(
+        stmt = "CREATE TEMPORARY TABLE {table_name} AS {custom_sql}".format(
             table_name=table_name, custom_sql=custom_sql)
         self.engine.execute(stmt)
 
     def _is_numeric_column(self, column):
         for col in self.columns:
             if (col['name'] == column and
-                isinstance(col['type'],
-                           (sa.types.Integer, sa.types.BigInteger, sa.types.Float, sa.types.Numeric, sa.types.SmallInteger, sa.types.Boolean)
-                           )
-            ):
+                    isinstance(col['type'],
+                               (sa.types.Integer, sa.types.BigInteger, sa.types.Float,
+                                sa.types.Numeric, sa.types.SmallInteger, sa.types.Boolean)
+                               )
+                ):
                 return True
 
         return False
@@ -275,9 +284,9 @@ def _is_numeric_column(self, column):
     @DocInherit
     @Dataset.expectation(['value'])
     def expect_table_row_count_to_equal(self,
-        value=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                        value=None,
+                                        result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                        ):
         # Assert that min_value and max_value are integers
         try:
             if value is not None:
@@ -289,7 +298,8 @@ def expect_table_row_count_to_equal(self,
         if value is None:
             raise ValueError("value must be provided")
 
-        count_query = sa.select([sa.func.count()]).select_from(self._table)
+        count_query = sa.select([sa.func.count()]).select_from(
+            self._table)
         row_count = self.engine.execute(count_query).scalar()
 
         return {
@@ -302,10 +312,10 @@ def expect_table_row_count_to_equal(self,
     @DocInherit
     @Dataset.expectation(['min_value', 'max_value'])
     def expect_table_row_count_to_be_between(self,
-        min_value=0,
-        max_value=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                             min_value=0,
+                                             max_value=None,
+                                             result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                             ):
         # Assert that min_value and max_value are integers
         try:
             if min_value is not None:
@@ -317,7 +327,8 @@ def expect_table_row_count_to_be_between(self,
         except ValueError:
             raise ValueError("min_value and max_value must be integers")
 
-        count_query = sa.select([sa.func.count()]).select_from(self._table)
+        count_query = sa.select([sa.func.count()]).select_from(
+            self._table)
         row_count = self.engine.execute(count_query).scalar()
 
         if min_value != None and max_value != None:
@@ -339,11 +350,11 @@ def expect_table_row_count_to_be_between(self,
     @DocInherit
     @Dataset.expectation(['column_list'])
     def expect_table_columns_to_match_ordered_list(self, column_list,
-                               result_format=None, include_config=False, catch_exceptions=None, meta=None):
+                                                   result_format=None, include_config=False, catch_exceptions=None, meta=None):
 
         if [col['name'] for col in self.columns] == list(column_list):
             return {
-                "success" : True
+                "success": True
             }
         else:
             return {
@@ -353,9 +364,9 @@ def expect_table_columns_to_match_ordered_list(self, column_list,
     @DocInherit
     @Dataset.expectation(['column'])
     def expect_column_to_exist(self,
-            column, column_index=None, result_format=None, include_config=False,
-            catch_exceptions=None, meta=None
-        ):
+                               column, column_index=None, result_format=None, include_config=False,
+                               catch_exceptions=None, meta=None
+                               ):
 
         col_names = [col['name'] for col in self.columns]
 
@@ -385,57 +396,57 @@ def expect_column_to_exist(self,
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_values_to_be_null(self,
-        column,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                        column,
+                                        mostly=None,
+                                        result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                        ):
 
         return sa.column(column) == None
 
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_values_to_not_be_null(self,
-        column,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                            column,
+                                            mostly=None,
+                                            result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                            ):
 
         return sa.column(column) != None
 
-
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_values_to_be_in_set(self,
-        column,
-        value_set,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
-        return sa.column(column).in_(tuple(value_set))
-
-    @DocInherit
-    @MetaSqlAlchemyDataset.column_map_expectation
-    def expect_column_values_to_not_be_in_set(self,
                                           column,
                                           value_set,
                                           mostly=None,
                                           result_format=None, include_config=False, catch_exceptions=None, meta=None
                                           ):
+        return sa.column(column).in_(tuple(value_set))
+
+    @DocInherit
+    @MetaSqlAlchemyDataset.column_map_expectation
+    def expect_column_values_to_not_be_in_set(self,
+                                              column,
+                                              value_set,
+                                              mostly=None,
+                                              result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                              ):
         return sa.column(column).notin_(tuple(value_set))
 
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_values_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        allow_cross_type_comparisons=None,
-        parse_strings_as_datetimes=None,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                           column,
+                                           min_value=None,
+                                           max_value=None,
+                                           allow_cross_type_comparisons=None,
+                                           parse_strings_as_datetimes=None,
+                                           mostly=None,
+                                           result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                           ):
         if parse_strings_as_datetimes is not None:
-            raise ValueError("parse_strings_as_datetimes is not currently supported in SqlAlchemy.")
+            raise ValueError(
+                "parse_strings_as_datetimes is not currently supported in SqlAlchemy.")
 
         if min_value != None and max_value != None and min_value > max_value:
             raise ValueError("min_value cannot be greater than max_value")
@@ -443,7 +454,6 @@ def expect_column_values_to_be_between(self,
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
 
-
         if min_value is None:
             return sa.column(column) <= max_value
 
@@ -452,29 +462,29 @@ def expect_column_values_to_be_between(self,
 
         else:
             return sa.and_(
-                        min_value <= sa.column(column),
-                        sa.column(column) <= max_value
-                )
+                min_value <= sa.column(column),
+                sa.column(column) <= max_value
+            )
 
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_value_lengths_to_equal(self,
-        column,
-        value,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                             column,
+                                             value,
+                                             mostly=None,
+                                             result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                             ):
         return sa.func.length(sa.column(column)) == value
 
     @DocInherit
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_value_lengths_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        mostly=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                                  column,
+                                                  min_value=None,
+                                                  max_value=None,
+                                                  mostly=None,
+                                                  result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                                  ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
@@ -513,22 +523,24 @@ def expect_column_value_lengths_to_be_between(self,
     @DocInherit
     @MetaSqlAlchemyDataset.column_aggregate_expectation
     def expect_column_max_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        parse_strings_as_datetimes=None,
-        output_strftime_format=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                        column,
+                                        min_value=None,
+                                        max_value=None,
+                                        parse_strings_as_datetimes=None,
+                                        output_strftime_format=None,
+                                        result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                        ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
 
         if parse_strings_as_datetimes:
-            raise ValueError("parse_strings_as_datetimes is not supported in SqlAlchemy")
+            raise ValueError(
+                "parse_strings_as_datetimes is not supported in SqlAlchemy")
 
         col_max = self.engine.execute(
-            sa.select([sa.func.max(sa.column(column))]).select_from(self._table)
+            sa.select([sa.func.max(sa.column(column))]).select_from(
+                self._table)
         ).scalar()
 
         # Handle possible missing values
@@ -556,26 +568,27 @@ def expect_column_max_to_be_between(self,
                 }
             }
 
-
     @DocInherit
     @MetaSqlAlchemyDataset.column_aggregate_expectation
     def expect_column_min_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        parse_strings_as_datetimes=None,
-        output_strftime_format=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                        column,
+                                        min_value=None,
+                                        max_value=None,
+                                        parse_strings_as_datetimes=None,
+                                        output_strftime_format=None,
+                                        result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                        ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
 
         if parse_strings_as_datetimes:
-            raise ValueError("parse_strings_as_datetimes is not supported in SqlAlchemy")
+            raise ValueError(
+                "parse_strings_as_datetimes is not supported in SqlAlchemy")
 
         col_min = self.engine.execute(
-            sa.select([sa.func.min(sa.column(column))]).select_from(self._table)
+            sa.select([sa.func.min(sa.column(column))]).select_from(
+                self._table)
         ).scalar()
 
         # Handle possible missing values
@@ -606,17 +619,18 @@ def expect_column_min_to_be_between(self,
     @DocInherit
     @MetaSqlAlchemyDataset.column_aggregate_expectation
     def expect_column_sum_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                        column,
+                                        min_value=None,
+                                        max_value=None,
+                                        result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                        ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
 
         col_sum = self.engine.execute(
-            sa.select([sa.func.sum(sa.column(column))]).select_from(self._table)
+            sa.select([sa.func.sum(sa.column(column))]).select_from(
+                self._table)
         ).scalar()
 
         # Handle possible missing values
@@ -647,11 +661,11 @@ def expect_column_sum_to_be_between(self,
     @DocInherit
     @MetaSqlAlchemyDataset.column_aggregate_expectation
     def expect_column_mean_to_be_between(self,
-        column,
-        min_value=None,
-        max_value=None,
-        result_format=None, include_config=False, catch_exceptions=None, meta=None
-    ):
+                                         column,
+                                         min_value=None,
+                                         max_value=None,
+                                         result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                         ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
@@ -666,7 +680,8 @@ def expect_column_mean_to_be_between(self,
             raise ValueError("column is not numeric")
 
         col_avg = self.engine.execute(
-            sa.select([sa.func.avg(sa.column(column))]).select_from(self._table)
+            sa.select([sa.func.avg(sa.column(column))]).select_from(
+                self._table)
         ).scalar()
 
         # Handle possible missing values
@@ -697,16 +712,15 @@ def expect_column_mean_to_be_between(self,
     @DocInherit
     @MetaSqlAlchemyDataset.column_aggregate_expectation
     def expect_column_median_to_be_between(self,
-                                         column,
-                                         min_value=None,
-                                         max_value=None,
-                                         result_format=None, include_config=False, catch_exceptions=None, meta=None
-                                         ):
+                                           column,
+                                           min_value=None,
+                                           max_value=None,
+                                           result_format=None, include_config=False, catch_exceptions=None, meta=None
+                                           ):
 
         if min_value is None and max_value is None:
             raise ValueError("min_value and max_value cannot both be None")
 
-
         # Inspiration from https://stackoverflow.com/questions/942620/missing-median-aggregate-function-in-django
         count_query = self.engine.execute(
             sa.select([
@@ -741,7 +755,7 @@ def expect_column_median_to_be_between(self,
                 'success': False,
                 'result': {
                     'observed_value': None
-                    }
+                }
             }
         else:
             if nonnull_count % 2 == 0:
@@ -761,7 +775,7 @@ def expect_column_median_to_be_between(self,
                 'result': {
                     'observed_value': column_median
                     }
-                }
+            }
 
     @MetaSqlAlchemyDataset.column_map_expectation
     def expect_column_values_to_be_unique(self, column, mostly=None,
@@ -784,7 +798,8 @@ def expect_column_unique_value_count_to_be_between(self, column, min_value=None,
             raise ValueError("min_value and max_value cannot both be None")
 
         unique_value_count = self.engine.execute(
-            sa.select([sa.func.count(sa.func.distinct(sa.column(column)))]).select_from(self._table)
+            sa.select([sa.func.count(sa.func.distinct(sa.column(column)))]).select_from(
+                self._table)
         ).scalar()
 
         # Handle possible missing values
@@ -797,7 +812,7 @@ def expect_column_unique_value_count_to_be_between(self, column, min_value=None,
             }
         else:
             return {
-                "success" : (
+                "success": (
                     ((min_value is None) or (min_value <= unique_value_count)) and
                     ((max_value is None) or (unique_value_count <= max_value))
                 ),
@@ -820,14 +835,16 @@ def expect_column_proportion_of_unique_values_to_be_between(self, column, min_va
                 sa.func.sum(
                     sa.case([(sa.column(column) == None, 1)], else_=0)
                 ).label('null_count'),
-                sa.func.count(sa.func.distinct(sa.column(column))).label('unique_value_count')
+                sa.func.count(sa.func.distinct(sa.column(column))
+                              ).label('unique_value_count')
             ]).select_from(self._table)
         )
 
         counts = count_query.fetchone()
 
         if counts['element_count'] - counts['null_count'] > 0:
-            proportion_unique = counts['unique_value_count'] / (counts['element_count'] - counts['null_count'])
+            proportion_unique = counts['unique_value_count'] / \
+                (counts['element_count'] - counts['null_count'])
         else:
             proportion_unique = None
 


Test patch: