Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/datasource/batch_kwargs_generator/s3_subdir_reader_batch_kwargs_generator.py b/great_expectations/datasource/batch_kwargs_generator/s3_subdir_reader_batch_kwargs_generator.py
--- a/great_expectations/datasource/batch_kwargs_generator/s3_subdir_reader_batch_kwargs_generator.py
+++ b/great_expectations/datasource/batch_kwargs_generator/s3_subdir_reader_batch_kwargs_generator.py
@@ -13,11 +13,7 @@
     BatchKwargsGenerator,
 )
 from great_expectations.datasource.types import PathBatchKwargs, S3BatchKwargs
-from great_expectations.exceptions import (
-    BatchKwargsError,
-    ClassInstantiationError,
-    InvalidConfigError,
-)
+from great_expectations.exceptions import BatchKwargsError
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +62,7 @@ def __init__(
         super().__init__(name, datasource=datasource)
 
         if not s3fs:
-            raise ClassInstantiationError("ModuleNotFoundError: No module named 's3fs'")
+            raise ImportError("ModuleNotFoundError: No module named 's3fs'")
 
         if reader_options is None:
             reader_options = self._default_reader_options
diff --git a/great_expectations/expectations/metrics/util.py b/great_expectations/expectations/metrics/util.py
--- a/great_expectations/expectations/metrics/util.py
+++ b/great_expectations/expectations/metrics/util.py
@@ -21,6 +21,7 @@
     import sqlalchemy as sa
     from sqlalchemy.dialects import registry
     from sqlalchemy.engine import Engine, reflection
+    from sqlalchemy.engine.interfaces import Dialect
     from sqlalchemy.exc import OperationalError
     from sqlalchemy.sql import Select
     from sqlalchemy.sql.elements import BinaryExpression, TextClause, literal
@@ -35,6 +36,7 @@
     custom_op = None
     Engine = None
     reflection = None
+    Dialect = None
     OperationalError = None
 
 try:
@@ -238,11 +240,12 @@ def get_sqlalchemy_column_metadata(
         return None
 
 
-def column_reflection_fallback(selectable, dialect, sqlalchemy_engine):
+def column_reflection_fallback(
+    selectable: Select, dialect: Dialect, sqlalchemy_engine: Engine
+) -> List[Dict[str, str]]:
     """If we can't reflect the table, use a query to at least get column names."""
-    col_info_dict_list: List[Dict]
+    col_info_dict_list: List[Dict[str, str]]
     if dialect.name.lower() == "mssql":
-        type_module = _get_dialect_type_module(dialect)
         # Get column names and types from the database
         # Reference: https://dataedo.com/kb/query/sql-server/list-table-columns-in-database
         columns_query: str = f"""
@@ -259,23 +262,29 @@ def column_reflection_fallback(selectable, dialect, sqlalchemy_engine):
     ON tab.object_id = col.object_id
     LEFT JOIN sys.types AS t
     ON col.user_type_id = t.user_type_id
+WHERE tab.name = '{selectable}'
 ORDER BY schema_name,
     table_name, 
     column_id
-            """
+"""
         col_info_query: TextClause = sa.text(columns_query)
         col_info_tuples_list: List[tuple] = sqlalchemy_engine.execute(
             col_info_query
         ).fetchall()
+        # type_module = _get_dialect_type_module(dialect=dialect)
         col_info_dict_list: List[Dict[str, str]] = [
-            {"name": column_name, "type": column_data_type}
+            {
+                "name": column_name,
+                # "type": getattr(type_module, column_data_type.upper())(),
+                "type": column_data_type.upper(),
+            }
             for schema_name, table_name, column_id, column_name, column_data_type, column_max_length, column_precision in col_info_tuples_list
         ]
     else:
         query: Select = sa.select([sa.text("*")]).select_from(selectable).limit(1)
         result_object = sqlalchemy_engine.execute(query)
         # noinspection PyProtectedMember
-        col_names = result_object._metadata.keys
+        col_names: List[str] = result_object._metadata.keys
         col_info_dict_list = [{"name": col_name} for col_name in col_names]
     return col_info_dict_list
 
diff --git a/great_expectations/self_check/util.py b/great_expectations/self_check/util.py
--- a/great_expectations/self_check/util.py
+++ b/great_expectations/self_check/util.py
@@ -313,9 +313,7 @@ def get_dataset(
                     df[col] = pd.to_datetime(df[col])
 
         if table_name is None:
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            table_name = generate_test_table_name()
         df.to_sql(
             name=table_name,
             con=engine,
@@ -376,9 +374,7 @@ def get_dataset(
                     df[col] = pd.to_datetime(df[col])
 
         if table_name is None:
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            table_name = generate_test_table_name()
         df.to_sql(
             name=table_name,
             con=engine,
@@ -435,9 +431,7 @@ def get_dataset(
                     df[col] = pd.to_datetime(df[col])
 
         if table_name is None:
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            table_name = generate_test_table_name()
         df.to_sql(
             name=table_name,
             con=engine,
@@ -507,9 +501,7 @@ def get_dataset(
                     df[col] = pd.to_datetime(df[col])
 
         if table_name is None:
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            table_name = generate_test_table_name()
         df.to_sql(
             name=table_name,
             con=engine,
@@ -670,9 +662,8 @@ def get_test_validator_with_data(
             df = df.astype(pandas_schema)
 
         if table_name is None:
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            # noinspection PyUnusedLocal
+            table_name = generate_test_table_name()
 
         return build_pandas_validator_with_data(df=df)
 
@@ -796,9 +787,7 @@ def get_test_validator_with_data(
 
         if table_name is None:
             # noinspection PyUnusedLocal
-            table_name = "test_data_" + "".join(
-                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-            )
+            table_name = generate_test_table_name()
 
         return build_spark_validator_with_data(df=spark_df, spark=spark)
 
@@ -892,9 +881,8 @@ def build_sa_validator_with_data(
                 df[col] = pd.to_datetime(df[col])
 
     if table_name is None:
-        table_name = "test_data_" + "".join(
-            [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
-        )
+        table_name = generate_test_table_name()
+
     df.to_sql(
         name=table_name,
         con=engine,
@@ -1952,3 +1940,12 @@ def check_json_test_result(test, result, data_asset=None):
                 raise ValueError(
                     "Invalid test specification: unknown key " + key + " in 'out'"
                 )
+
+
+def generate_test_table_name(
+    default_table_name_prefix: Optional[str] = "test_data_",
+) -> str:
+    table_name: str = default_table_name_prefix + "".join(
+        [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
+    )
+    return table_name


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2287,7 +2287,6 @@ def empty_data_context(tmp_path) -> DataContext:
 @pytest.fixture
 def titanic_pandas_data_context_with_v013_datasource_with_checkpoints_v1_with_empty_store_stats_enabled(
     tmp_path_factory,
-    test_backends,
     monkeypatch,
 ):
     # Reenable GE_USAGE_STATS
diff --git a/tests/datasource/batch_kwarg_generator/test_s3_subdir_reader_generator.py b/tests/datasource/batch_kwarg_generator/test_s3_subdir_reader_generator.py
--- a/tests/datasource/batch_kwarg_generator/test_s3_subdir_reader_generator.py
+++ b/tests/datasource/batch_kwarg_generator/test_s3_subdir_reader_generator.py
@@ -1,4 +1,3 @@
-import logging
 import os
 import time
 
@@ -10,7 +9,6 @@
 from great_expectations.datasource.batch_kwargs_generator import (
     S3SubdirReaderBatchKwargsGenerator,
 )
-from great_expectations.exceptions import BatchKwargsError
 
 port = 5555
 url_host = os.getenv("GE_TEST_LOCALHOST_URL", "127.0.0.1")
@@ -65,28 +63,34 @@ def mock_s3_bucket(s3_base):
 def s3_subdir_generator(mock_s3_bucket, basic_sparkdf_datasource):
     # We configure a generator that will fetch from (mocked) my_bucket
     # and will use glob patterns to match returned assets into batches of the same asset
-    generator = S3SubdirReaderBatchKwargsGenerator(
-        "my_generator",
-        datasource=basic_sparkdf_datasource,
-        boto3_options={"endpoint_url": endpoint_uri},
-        base_directory="test_bucket/data/for",
-        reader_options={"sep": ","},
-    )
-    yield generator
+    try:
+        generator = S3SubdirReaderBatchKwargsGenerator(
+            "my_generator",
+            datasource=basic_sparkdf_datasource,
+            boto3_options={"endpoint_url": endpoint_uri},
+            base_directory="test_bucket/data/for",
+            reader_options={"sep": ","},
+        )
+        yield generator
+    except ImportError as e:
+        pytest.skip(str(e))
 
 
 @pytest.fixture
 def s3_subdir_generator_with_partition(mock_s3_bucket, basic_sparkdf_datasource):
     # We configure a generator that will fetch from (mocked) my_bucket
     # and will use glob patterns to match returned assets into batches of the same asset
-    generator = S3SubdirReaderBatchKwargsGenerator(
-        "my_generator",
-        datasource=basic_sparkdf_datasource,
-        boto3_options={"endpoint_url": endpoint_uri},
-        base_directory="test_bucket/data/",
-        reader_options={"sep": ","},
-    )
-    yield generator
+    try:
+        generator = S3SubdirReaderBatchKwargsGenerator(
+            "my_generator",
+            datasource=basic_sparkdf_datasource,
+            boto3_options={"endpoint_url": endpoint_uri},
+            base_directory="test_bucket/data/",
+            reader_options={"sep": ","},
+        )
+        yield generator
+    except ImportError as e:
+        pytest.skip(str(e))
 
 
 def test_s3_subdir_generator_basic_operation(s3_subdir_generator):
diff --git a/tests/expectations/test_util.py b/tests/expectations/test_util.py
--- a/tests/expectations/test_util.py
+++ b/tests/expectations/test_util.py
@@ -1,11 +1,75 @@
+import logging
+from typing import Dict, List
+
+import pandas as pd
 import pytest
 
-from great_expectations.core import ExpectationConfiguration
+from great_expectations.core import (
+    ExpectationConfiguration,
+    ExpectationValidationResult,
+)
 from great_expectations.exceptions import GreatExpectationsError
 from great_expectations.execution_engine import ExecutionEngine
+from great_expectations.expectations.metrics.util import column_reflection_fallback
 from great_expectations.expectations.util import render_evaluation_parameter_string
 from great_expectations.render.types import RenderedStringTemplateContent
+from great_expectations.self_check.util import (
+    build_sa_validator_with_data,
+    build_test_backends_list,
+    generate_test_table_name,
+)
 from great_expectations.validator.validation_graph import MetricConfiguration
+from great_expectations.validator.validator import Validator
+
+logger = logging.getLogger(__name__)
+
+try:
+    import sqlalchemy as sqlalchemy
+    from sqlalchemy import create_engine
+
+    # noinspection PyProtectedMember
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import SQLAlchemyError
+    from sqlalchemy.sql import Select
+except ImportError:
+    sqlalchemy = None
+    create_engine = None
+    Engine = None
+    Select = None
+    SQLAlchemyError = None
+    logger.debug("Unable to load SqlAlchemy or one of its subclasses.")
+
+
+def get_table_columns_metric(engine: ExecutionEngine) -> [MetricConfiguration, dict]:
+    resolved_metrics: dict = {}
+
+    results: dict
+
+    table_column_types_metric: MetricConfiguration = MetricConfiguration(
+        metric_name="table.column_types",
+        metric_domain_kwargs=dict(),
+        metric_value_kwargs={
+            "include_nested": True,
+        },
+        metric_dependencies=None,
+    )
+    results = engine.resolve_metrics(metrics_to_resolve=(table_column_types_metric,))
+    resolved_metrics.update(results)
+
+    table_columns_metric: MetricConfiguration = MetricConfiguration(
+        metric_name="table.columns",
+        metric_domain_kwargs=dict(),
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "table.column_types": table_column_types_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(table_columns_metric,), metrics=resolved_metrics
+    )
+    resolved_metrics.update(results)
+
+    return table_columns_metric, resolved_metrics
 
 
 @pytest.fixture(scope="module")
@@ -251,33 +315,95 @@ def bare_bones_prescriptive_renderer(
     assert len(res) == 2
 
 
-def get_table_columns_metric(engine: ExecutionEngine) -> [MetricConfiguration, dict]:
-    resolved_metrics: dict = {}
+def test_table_column_reflection_fallback(test_backends, sa):
+    include_sqlalchemy: bool = "sqlite" in test_backends
+    include_postgresql: bool = "postgresql" in test_backends
+    include_mysql: bool = "mysql" in test_backends
+    include_mssql: bool = "mssql" in test_backends
 
-    results: dict
+    if not create_engine:
+        pytest.skip("Unable to import sqlalchemy.create_engine() -- skipping.")
 
-    table_column_types_metric: MetricConfiguration = MetricConfiguration(
-        metric_name="table.column_types",
-        metric_domain_kwargs=dict(),
-        metric_value_kwargs={
-            "include_nested": True,
-        },
-        metric_dependencies=None,
+    execution_engine_names: List[str] = build_test_backends_list(
+        include_pandas=False,
+        include_spark=False,
+        include_sqlalchemy=include_sqlalchemy,
+        include_postgresql=include_postgresql,
+        include_mysql=include_mysql,
+        include_mssql=include_mssql,
     )
-    results = engine.resolve_metrics(metrics_to_resolve=(table_column_types_metric,))
-    resolved_metrics.update(results)
 
-    table_columns_metric: MetricConfiguration = MetricConfiguration(
-        metric_name="table.columns",
-        metric_domain_kwargs=dict(),
-        metric_value_kwargs=None,
-        metric_dependencies={
-            "table.column_types": table_column_types_metric,
-        },
-    )
-    results = engine.resolve_metrics(
-        metrics_to_resolve=(table_columns_metric,), metrics=resolved_metrics
+    df: pd.DataFrame = pd.DataFrame(
+        {
+            "name": ["Frank", "Steve", "Jane", "Frank", "Michael"],
+            "age": [16, 21, 38, 22, 10],
+            "pet": ["fish", "python", "cat", "python", "frog"],
+        }
     )
-    resolved_metrics.update(results)
 
-    return table_columns_metric, resolved_metrics
+    validators_config: Dict[str, Validator] = {}
+    validator: Validator
+    table_name: str
+    for execution_engine_name in execution_engine_names:
+        if execution_engine_name in ["sqlite", "postgresql", "mysql", "mssql"]:
+            table_name = generate_test_table_name()
+            validator = build_sa_validator_with_data(
+                df=df,
+                sa_engine_name=execution_engine_name,
+                schemas=None,
+                caching=True,
+                table_name=table_name,
+                sqlite_db_path=None,
+            )
+            if validator is not None:
+                validators_config[table_name] = validator
+
+    engine: Engine
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    reflected_columns_list: List[Dict[str, str]]
+    reflected_column_config: Dict[str, str]
+    column_name: str
+
+    validation_result: ExpectationValidationResult
+
+    for table_name, validator in validators_config.items():
+        table_columns_metric, results = get_table_columns_metric(
+            engine=validator.execution_engine
+        )
+        metrics.update(results)
+        assert set(metrics[table_columns_metric.id]) == {"name", "age", "pet"}
+        selectable: Select = sqlalchemy.Table(
+            table_name,
+            sqlalchemy.MetaData(),
+            schema=None,
+        )
+        reflected_columns_list = column_reflection_fallback(
+            selectable=selectable,
+            dialect=validator.execution_engine.engine.dialect,
+            sqlalchemy_engine=validator.execution_engine.engine,
+        )
+        for column_name in [
+            reflected_column_config["name"]
+            for reflected_column_config in reflected_columns_list
+        ]:
+            validation_result = validator.expect_column_to_exist(column=column_name)
+            assert validation_result.success
+
+    if validators_config:
+        validator = list(validators_config.values())[0]
+
+        validation_result = validator.expect_column_mean_to_be_between(
+            column="age", min_value=10
+        )
+        assert validation_result.success
+
+        validation_result = validator.expect_table_row_count_to_equal(value=5)
+        assert validation_result.success
+
+        validation_result = validator.expect_table_row_count_to_equal(value=3)
+        assert not validation_result.success