Repository: great-expectations/great_expectations

Issue:
Invalid Temp Table SQL for MSSQL
There looks to be similar open issue (#1012), but opening this for the specific invalid syntax for temp tables with mssql. 

When following great_expectations init and using a relational database option with MSSQL connection, the following error occurs:

```
 "[42000] [Microsoft][ODBC Driver 13 for SQL Server][SQL Server]Unknown object type 'TEMPORARY' used in a CREATE, DROP, or ALTER statement. (343) (SQLExecDirectW)")
```

I'll see if I can suggest a PR to fix this.



Discussion:
I encountered the same problem. I think it can be fixed by adding another elif clause to create_temporary_table() in sqlalchemy_dataset.py as follows:

```
elif self.engine.dialect.name == "mssql":
     custom_sqlmod = custom_sql.split('from')
     stmt = (custom_sqlmod[0] + "into #{table_name} from" + custom_sqlmod[1]).format(table_name=table_name)
```

Temporary tables in MSSQL are indicated by `#`.

Resolving patch:
diff --git a/great_expectations/dataset/sqlalchemy_dataset.py b/great_expectations/dataset/sqlalchemy_dataset.py
--- a/great_expectations/dataset/sqlalchemy_dataset.py
+++ b/great_expectations/dataset/sqlalchemy_dataset.py
@@ -253,6 +253,9 @@ def __init__(self, table_name=None, engine=None, connection_string=None,
             #was used for a temp table and raising an error
             schema = None
             table_name = "ge_tmp_" + str(uuid.uuid4())[:8]
+            # mssql expects all temporary table names to have a prefix '#'
+            if engine.dialect.name.lower() == "mssql":
+                table_name = "#" + table_name
             generated_table_name = table_name
         else:
             generated_table_name = None
@@ -279,8 +282,8 @@ def __init__(self, table_name=None, engine=None, connection_string=None,
             # These are the officially included and supported dialects by sqlalchemy
             self.dialect = import_module("sqlalchemy.dialects." + self.engine.dialect.name)
 
-            if engine and engine.dialect.name.lower() == "sqlite":
-                # sqlite temp tables only persist within a connection so override the engine
+            if engine and engine.dialect.name.lower() in ["sqlite", "mssql"]:
+                # sqlite/mssql temp tables only persist within a connection so override the engine
                 self.engine = engine.connect()
         elif self.engine.dialect.name.lower() == "snowflake":
             self.dialect = import_module("snowflake.sqlalchemy.snowdialect")
@@ -325,6 +328,10 @@ def __init__(self, table_name=None, engine=None, connection_string=None,
             # reflection will not find the temporary schema
             self.columns = self.column_reflection_fallback()
 
+        # Use fallback because for mssql reflection doesn't throw an error but returns an empty list
+        if len(self.columns) == 0:
+            self.columns = self.column_reflection_fallback()
+
         # Only call super once connection is established and table_name and columns known to allow autoinspection
         super(SqlAlchemyDataset, self).__init__(*args, **kwargs)
 
@@ -351,6 +358,10 @@ def head(self, n=5):
                 head_sql_str += self._table.name
             head_sql_str += " limit {0:d}".format(n)
 
+            # Limit is unknown in mssql! Use top instead!
+            if self.engine.dialect.name.lower() == "mssql":
+                head_sql_str = "select top({n}) * from {table}".format(n=n, table=self._table.name)
+
             df = pd.read_sql(head_sql_str, con=self.engine)
 
         return PandasDataset(
@@ -482,8 +493,13 @@ def get_column_median(self, column):
         return column_median
 
     def get_column_quantiles(self, column, quantiles, allow_relative_error=False):
-        selects = [sa.func.percentile_disc(quantile).within_group(
-            sa.column(column).asc()) for quantile in quantiles]
+        if self.engine.dialect.name.lower() != "mssql":
+            selects = [sa.func.percentile_disc(quantile).within_group(
+                sa.column(column).asc()) for quantile in quantiles]
+        else:
+            # mssql requires over(), so we add an empty over() clause
+            selects = [sa.func.percentile_disc(quantile).within_group(
+                sa.column(column).asc()).over() for quantile in quantiles]
         try:
             if isinstance(self.engine.dialect, sqlalchemy_redshift.dialect.RedshiftDialect):
                 # Redshift does not have a percentile_disc method, but does support an approximate version
@@ -501,9 +517,15 @@ def get_column_quantiles(self, column, quantiles, allow_relative_error=False):
         return list(quantiles)
 
     def get_column_stdev(self, column):
-        res = self.engine.execute(sa.select([
-                sa.func.stddev_samp(sa.column(column))
-            ]).select_from(self._table).where(sa.column(column) != None)).fetchone()
+        if self.engine.dialect.name.lower() != "mssql":
+            res = self.engine.execute(sa.select([
+                    sa.func.stddev_samp(sa.column(column))
+                ]).select_from(self._table).where(sa.column(column) != None)).fetchone()
+        else:
+            # stdev_samp is not a recognized built-in function name but stdevp does exist for mssql!
+            res = self.engine.execute(sa.select([
+                    sa.func.stdevp(sa.column(column))
+                ]).select_from(self._table).where(sa.column(column) != None)).fetchone()
         return float(res[0])
 
     def get_column_hist(self, column, bins):
@@ -664,6 +686,16 @@ def create_temporary_table(self, table_name, custom_sql, schema_name=None):
         elif self.engine.dialect.name == "mysql":
             stmt = "CREATE TEMPORARY TABLE {table_name} AS {custom_sql}".format(
                 table_name=table_name, custom_sql=custom_sql)
+        elif self.engine.dialect.name == "mssql":
+            # Insert "into #{table_name}" in the custom sql query right before the "from" clause
+            # Split is case sensitive so detect case.
+            # Note: transforming custom_sql to uppercase/lowercase has uninteded consequences (i.e., changing column names), so this is not an option!
+            if 'from' in custom_sql:
+                strsep = 'from'
+            else:
+                strsep = 'FROM'
+            custom_sqlmod = custom_sql.split(strsep, maxsplit=1)
+            stmt = (custom_sqlmod[0] + "into {table_name} from" + custom_sqlmod[1]).format(table_name=table_name)
         else:
             stmt = "CREATE TEMPORARY TABLE \"{table_name}\" AS {custom_sql}".format(
                 table_name=table_name, custom_sql=custom_sql)
@@ -671,9 +703,16 @@ def create_temporary_table(self, table_name, custom_sql, schema_name=None):
 
     def column_reflection_fallback(self):
         """If we can't reflect the table, use a query to at least get column names."""
-        sql = sa.select([sa.text("*")]).select_from(self._table).limit(1)
-        col_names = self.engine.execute(sql).keys()
-        col_dict = [{'name': col_name} for col_name in col_names]
+        if self.engine.dialect.name.lower() != "mssql":
+            sql = sa.select([sa.text("*")]).select_from(self._table).limit(1)
+            col_names = self.engine.execute(sql).keys()
+            col_dict = [{'name': col_name} for col_name in col_names]
+        else:
+            type_module = self._get_dialect_type_module()
+            # Get column names and types from the database
+            # StackOverflow to the rescue: https://stackoverflow.com/a/38634368
+            col_info = self.engine.execute("SELECT cols.NAME,ty.NAME FROM tempdb.sys.columns cols JOIN sys.types ty ON cols.user_type_id = ty.user_type_id WHERE object_id = OBJECT_ID('tempdb..{}')".format(self._table)).fetchall()
+            col_dict = [{'name': col_name, 'type': getattr(type_module,col_type.upper())()} for col_name,col_type in col_info]
         return col_dict
 
     ###


Test patch: