Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/validator/validator.py b/great_expectations/validator/validator.py
--- a/great_expectations/validator/validator.py
+++ b/great_expectations/validator/validator.py
@@ -92,16 +92,10 @@ def __init__(
             batches = tuple()
 
         self._batches = {}
-
-        for batch in batches:
-            assert isinstance(
-                batch, Batch
-            ), "batches provided to Validator must be Great Expectations Batch objects"
-            self._execution_engine.load_batch_data(batch.id, batch.data)
-            self._batches[batch.id] = batch
+        self.load_batch_list(batches)
 
         if len(batches) > 1:
-            logger.warning(
+            logger.debug(
                 f"{len(batches)} batches will be added to this Validator. The batch_identifiers for the active "
                 f"batch are {self.active_batch.batch_definition['batch_identifiers'].items()}"
             )
@@ -437,9 +431,12 @@ def graph_validate(
             except AssertionError as e:
                 raise InvalidExpectationConfigurationError(str(e))
 
-            expectation_impl = get_expectation_impl(configuration.expectation_type)
+            evaluated_config = copy.deepcopy(configuration)
+            evaluated_config.kwargs.update({"batch_id": self.active_batch_id})
+
+            expectation_impl = get_expectation_impl(evaluated_config.expectation_type)
             validation_dependencies = expectation_impl().get_validation_dependencies(
-                configuration, self._execution_engine, runtime_configuration
+                evaluated_config, self._execution_engine, runtime_configuration
             )["metrics"]
 
             try:
@@ -447,11 +444,11 @@ def graph_validate(
                     self.build_metric_dependency_graph(
                         graph,
                         metric,
-                        configuration,
+                        evaluated_config,
                         self._execution_engine,
                         runtime_configuration=runtime_configuration,
                     )
-                processed_configurations.append(configuration)
+                processed_configurations.append(evaluated_config)
             except Exception as err:
                 if catch_exceptions:
                     raised_exception = True
@@ -463,7 +460,7 @@ def graph_validate(
                             "exception_traceback": exception_traceback,
                             "exception_message": str(err),
                         },
-                        expectation_config=configuration,
+                        expectation_config=evaluated_config,
                     )
                     evrs.append(result)
                 else:
@@ -692,8 +689,14 @@ def get_config_value(self, key):
         """Getter for config value"""
         return self._validator_config.get(key)
 
-    def load_batch(self, batch_list: List[Batch]):
+    def load_batch_list(self, batch_list: List[Batch]):
         for batch in batch_list:
+            try:
+                assert isinstance(
+                    batch, Batch
+                ), "batches provided to Validator must be Great Expectations Batch objects"
+            except AssertionError as e:
+                logger.warning(str(e))
             self._execution_engine.load_batch_data(batch.id, batch.data)
             self._batches[batch.id] = batch
             # We set the active_batch_id in each iteration of the loop to keep in sync with the active_batch_id for the
@@ -714,7 +717,7 @@ def loaded_batch_ids(self) -> List[str]:
     @property
     def active_batch(self) -> Batch:
         """Getter for active batch"""
-        active_batch_id: str = self.execution_engine.active_batch_data_id
+        active_batch_id: str = self.active_batch_id
         batch: Batch = self.batches.get(active_batch_id) if active_batch_id else None
         return batch
 
@@ -729,10 +732,15 @@ def active_batch_spec(self) -> Optional[BatchSpec]:
     @property
     def active_batch_id(self) -> str:
         """Getter for active batch id"""
-        return self.execution_engine.active_batch_data_id
+        active_engine_batch_id = self._execution_engine.active_batch_data_id
+        if active_engine_batch_id != self._active_batch_id:
+            logger.debug(
+                "This validator has a different active batch id than its Execution Engine."
+            )
+        return self._active_batch_id
 
     @active_batch_id.setter
-    def active_batch_id(self, batch_id: str):
+    def active_batch_id(self, batch_id):
         assert set(self.batches.keys()).issubset(set(self.loaded_batch_ids))
         available_batch_ids: Set[str] = set(self.batches.keys()).union(
             set(self.loaded_batch_ids)
@@ -744,7 +752,7 @@ def active_batch_id(self, batch_id: str):
 """
             )
         else:
-            self.execution_engine._active_batch_data_id = batch_id
+            self._active_batch_id = batch_id
 
     @property
     def active_batch_markers(self):


Test patch:
diff --git a/tests/datasource/data_connector/test_sql_data_connector.py b/tests/datasource/data_connector/test_sql_data_connector.py
--- a/tests/datasource/data_connector/test_sql_data_connector.py
+++ b/tests/datasource/data_connector/test_sql_data_connector.py
@@ -5,7 +5,7 @@
 from packaging.version import parse as parse_version
 from ruamel.yaml import YAML
 
-from great_expectations.core.batch import BatchRequest
+from great_expectations.core.batch import Batch, BatchRequest
 from great_expectations.core.batch_spec import SqlAlchemyDatasourceBatchSpec
 from great_expectations.data_context.util import instantiate_class_from_config
 from great_expectations.datasource.data_connector import ConfiguredAssetSqlDataConnector
@@ -572,8 +572,10 @@ def test_sampling_method__limit(
             }
         )
     )
-    execution_engine.load_batch_data("__", batch_data)
-    validator = Validator(execution_engine)
+
+    batch = Batch(data=batch_data)
+
+    validator = Validator(execution_engine, batches=[batch])
     assert len(validator.head(fetch_all=True)) == 20
 
     assert (
diff --git a/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py b/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
--- a/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
+++ b/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
@@ -1,36 +1,19 @@
 import pandas as pd
+import pytest
 
-from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.core.expectation_validation_result import (
     ExpectationValidationResult,
 )
-from great_expectations.core.util import get_or_create_spark_application
-from great_expectations.execution_engine import PandasExecutionEngine
-from great_expectations.execution_engine.sqlalchemy_execution_engine import (
-    SqlAlchemyBatchData,
-    SqlAlchemyExecutionEngine,
+from great_expectations.self_check.util import (
+    build_pandas_validator_with_data,
+    build_sa_validator_with_data,
+    build_spark_validator_with_data,
 )
-from great_expectations.expectations.core.expect_column_value_z_scores_to_be_less_than import (
-    ExpectColumnValueZScoresToBeLessThan,
-)
-from great_expectations.validator.validator import Validator
 
 
-def test_pandas_expect_column_value_z_scores_to_be_less_than_impl():
-    df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
-    expectationConfiguration = ExpectationConfiguration(
-        expectation_type="expect_column_value_z_scores_to_be_less_than",
-        kwargs={
-            "column": "a",
-            "mostly": 0.9,
-            "threshold": 4,
-            "double_sided": True,
-        },
-    )
-    expectation = ExpectColumnValueZScoresToBeLessThan(expectationConfiguration)
-    engine = PandasExecutionEngine(batch_data_dict={"my_id": df})
-    result = expectation.validate(Validator(execution_engine=engine))
-    assert result == ExpectationValidationResult(
+@pytest.fixture
+def z_score_validation_result():
+    return ExpectationValidationResult(
         success=True,
         expectation_config={
             "expectation_type": "expect_column_value_z_scores_to_be_less_than",
@@ -61,113 +44,46 @@ def test_pandas_expect_column_value_z_scores_to_be_less_than_impl():
     )
 
 
-def test_sa_expect_column_value_z_scores_to_be_less_than_impl(postgresql_engine):
+def test_pandas_expect_column_value_z_scores_to_be_less_than_impl(
+    z_score_validation_result,
+):
     df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
-    df.to_sql(
-        name="z_score_test_data",
-        con=postgresql_engine,
-        index=False,
-        if_exists="replace",
-    )
-    expectationConfiguration = ExpectationConfiguration(
-        expectation_type="expect_column_value_z_scores_to_be_less_than",
-        kwargs={
-            "column": "a",
-            "mostly": 0.9,
-            "threshold": 4,
-            "double_sided": True,
-        },
-    )
-    expectation = ExpectColumnValueZScoresToBeLessThan(expectationConfiguration)
-    engine = SqlAlchemyExecutionEngine(engine=postgresql_engine)
-    engine.load_batch_data(
-        "my_id",
-        SqlAlchemyBatchData(execution_engine=engine, table_name="z_score_test_data"),
+
+    validator = build_pandas_validator_with_data(df)
+
+    result = validator.expect_column_value_z_scores_to_be_less_than(
+        column="a", mostly=0.9, threshold=4, double_sided=True
     )
-    result = expectation.validate(Validator(execution_engine=engine))
-    assert result == ExpectationValidationResult(
-        success=True,
-        expectation_config={
-            "expectation_type": "expect_column_value_z_scores_to_be_less_than",
-            "kwargs": {
-                "column": "a",
-                "mostly": 0.9,
-                "threshold": 4,
-                "double_sided": True,
-            },
-            "meta": {},
-        },
-        result={
-            "element_count": 6,
-            "unexpected_count": 0,
-            "unexpected_percent": 0.0,
-            "partial_unexpected_list": [],
-            "missing_count": 0,
-            "missing_percent": 0.0,
-            "unexpected_percent_total": 0.0,
-            "unexpected_percent_nonmissing": 0.0,
-        },
-        exception_info={
-            "raised_exception": False,
-            "exception_traceback": None,
-            "exception_message": None,
-        },
-        meta={},
+
+    assert result == z_score_validation_result
+
+
+def test_sa_expect_column_value_z_scores_to_be_less_than_impl(
+    z_score_validation_result, test_backends
+):
+    if "postgresql" not in test_backends:
+        pytest.skip("test_database_store_backend_get_url_for_key requires postgresql")
+
+    df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
+
+    validator = build_sa_validator_with_data(df=df, sa_engine_name="postgresql")
+
+    result = validator.expect_column_value_z_scores_to_be_less_than(
+        column="a", mostly=0.9, threshold=4, double_sided=True
     )
 
+    assert result == z_score_validation_result
+
 
 def test_spark_expect_column_value_z_scores_to_be_less_than_impl(
-    spark_session, basic_spark_df_execution_engine
+    spark_session, basic_spark_df_execution_engine, z_score_validation_result
 ):
     df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
-    spark = get_or_create_spark_application(
-        spark_config={
-            "spark.sql.catalogImplementation": "hive",
-            "spark.executor.memory": "450m",
-            # "spark.driver.allowMultipleContexts": "true",  # This directive does not appear to have any effect.
-        }
-    )
-    df = spark.createDataFrame(df)
-
-    expectationConfiguration = ExpectationConfiguration(
-        expectation_type="expect_column_value_z_scores_to_be_less_than",
-        kwargs={
-            "column": "a",
-            "mostly": 0.9,
-            "threshold": 4,
-            "double_sided": True,
-        },
-    )
-    expectation = ExpectColumnValueZScoresToBeLessThan(expectationConfiguration)
-    engine = basic_spark_df_execution_engine
-    engine.load_batch_data(batch_id="my_id", batch_data=df)
-    result = expectation.validate(Validator(execution_engine=engine))
-    assert result == ExpectationValidationResult(
-        success=True,
-        expectation_config={
-            "expectation_type": "expect_column_value_z_scores_to_be_less_than",
-            "kwargs": {
-                "column": "a",
-                "mostly": 0.9,
-                "threshold": 4,
-                "double_sided": True,
-            },
-            "meta": {},
-        },
-        result={
-            "element_count": 6,
-            "unexpected_count": 0,
-            "unexpected_percent": 0.0,
-            "partial_unexpected_list": [],
-            "missing_count": 0,
-            "missing_percent": 0.0,
-            "unexpected_percent_total": 0.0,
-            "unexpected_percent_nonmissing": 0.0,
-        },
-        exception_info={
-            "raised_exception": False,
-            "exception_traceback": None,
-            "exception_message": None,
-        },
-        meta={},
+
+    validator = build_spark_validator_with_data(df, spark_session)
+
+    result = validator.expect_column_value_z_scores_to_be_less_than(
+        column="a", mostly=0.9, threshold=4, double_sided=True
     )
+
+    assert result == z_score_validation_result
diff --git a/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py b/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
--- a/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
+++ b/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
@@ -1,45 +1,23 @@
 import pandas as pd
 
-from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.core.expectation_validation_result import (
     ExpectationValidationResult,
 )
-from great_expectations.execution_engine.sqlalchemy_execution_engine import (
-    SqlAlchemyBatchData,
-    SqlAlchemyExecutionEngine,
-)
-from great_expectations.expectations.core.expect_column_values_to_be_in_type_list import (
-    ExpectColumnValuesToBeInTypeList,
-)
-from great_expectations.validator.validator import Validator
+from great_expectations.self_check.util import build_sa_validator_with_data
 
 
 def test_expect_column_values_to_be_in_type_list_dialect_pyathena(sa):
     from pyathena import sqlalchemy_athena
 
-    engine = sa.create_engine("sqlite://")
-
-    data = pd.DataFrame({"col": ["test_val1", "test_val2"]})
-
-    data.to_sql(name="test_sql_data", con=engine, index=False)
+    df = pd.DataFrame({"col": ["test_val1", "test_val2"]})
+    validator = build_sa_validator_with_data(df, "sqlite")
 
-    expectationConfiguration = ExpectationConfiguration(
-        expectation_type="expect_column_values_to_be_in_type_list",
-        kwargs={
-            "column": "col",
-            "type_list": ["STRINGTYPE", "BOOLEAN"],
-        },
-    )
-    expectation = ExpectColumnValuesToBeInTypeList(expectationConfiguration)
-    engine = SqlAlchemyExecutionEngine(engine=engine)
     # Monkey-patch dialect for testing purposes.
-    engine.dialect_module = sqlalchemy_athena
-    engine.load_batch_data(
-        "my_id",
-        SqlAlchemyBatchData(execution_engine=engine, table_name="test_sql_data"),
-    )
-    result = expectation.validate(Validator(execution_engine=engine))
+    validator.execution_engine.dialect_module = sqlalchemy_athena
 
+    result = validator.expect_column_values_to_be_in_type_list(
+        "col", type_list=["STRINGTYPE", "BOOLEAN"]
+    )
     assert result == ExpectationValidationResult(
         success=True,
         expectation_config={
diff --git a/tests/expectations/core/test_expect_column_values_to_be_of_type.py b/tests/expectations/core/test_expect_column_values_to_be_of_type.py
--- a/tests/expectations/core/test_expect_column_values_to_be_of_type.py
+++ b/tests/expectations/core/test_expect_column_values_to_be_of_type.py
@@ -1,44 +1,21 @@
 import pandas as pd
 
-from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.core.expectation_validation_result import (
     ExpectationValidationResult,
 )
-from great_expectations.execution_engine.sqlalchemy_execution_engine import (
-    SqlAlchemyBatchData,
-    SqlAlchemyExecutionEngine,
-)
-from great_expectations.expectations.core.expect_column_values_to_be_of_type import (
-    ExpectColumnValuesToBeOfType,
-)
-from great_expectations.validator.validator import Validator
+from great_expectations.self_check.util import build_sa_validator_with_data
 
 
 def test_expect_column_values_to_be_of_type_string_dialect_pyathena(sa):
     from pyathena import sqlalchemy_athena
 
-    engine = sa.create_engine("sqlite://")
-
-    data = pd.DataFrame({"col": ["test_val1", "test_val2"]})
-
-    data.to_sql(name="test_sql_data", con=engine, index=False)
+    df = pd.DataFrame({"col": ["test_val1", "test_val2"]})
+    validator = build_sa_validator_with_data(df, "sqlite")
 
-    expectationConfiguration = ExpectationConfiguration(
-        expectation_type="expect_column_values_to_be_of_type",
-        kwargs={
-            "column": "col",
-            "type_": "STRINGTYPE",
-        },
-    )
-    expectation = ExpectColumnValuesToBeOfType(expectationConfiguration)
-    engine = SqlAlchemyExecutionEngine(engine=engine)
     # Monkey-patch dialect for testing purposes.
-    engine.dialect_module = sqlalchemy_athena
-    engine.load_batch_data(
-        "my_id",
-        SqlAlchemyBatchData(execution_engine=engine, table_name="test_sql_data"),
-    )
-    result = expectation.validate(Validator(execution_engine=engine))
+    validator.execution_engine.dialect_module = sqlalchemy_athena
+
+    result = validator.expect_column_values_to_be_of_type("col", type_="STRINGTYPE")
 
     assert result == ExpectationValidationResult(
         success=True,
diff --git a/tests/integration/fixtures/yellow_trip_data_pandas_fixture/multiple_batch_requests_one_validator_multiple_steps.py b/tests/integration/fixtures/yellow_trip_data_pandas_fixture/multiple_batch_requests_one_validator_multiple_steps.py
--- a/tests/integration/fixtures/yellow_trip_data_pandas_fixture/multiple_batch_requests_one_validator_multiple_steps.py
+++ b/tests/integration/fixtures/yellow_trip_data_pandas_fixture/multiple_batch_requests_one_validator_multiple_steps.py
@@ -38,7 +38,7 @@
 
 feb_batch_list: List[Batch] = context.get_batch_list(batch_request=feb_batch_request)
 
-validator.load_batch(batch_list=feb_batch_list)
+validator.load_batch_list(batch_list=feb_batch_list)
 assert validator.active_batch_definition.batch_identifiers["month"] == "02"
 assert validator.active_batch_definition.batch_identifiers["year"] == "2019"
 
@@ -54,7 +54,7 @@
     batch_request=march_batch_request
 )
 
-validator.load_batch(batch_list=march_batch_list)
+validator.load_batch_list(batch_list=march_batch_list)
 assert validator.active_batch_definition.batch_identifiers["month"] == "03"
 assert validator.active_batch_definition.batch_identifiers["year"] == "2019"
 
diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py
--- a/tests/validator/test_validator.py
+++ b/tests/validator/test_validator.py
@@ -704,7 +704,7 @@ def test_validator_load_additional_batch_to_validator(
     )
 
     new_batch = context.get_batch_list(batch_request=feb_batch_request)
-    validator.load_batch(batch_list=new_batch)
+    validator.load_batch_list(batch_list=new_batch)
 
     updated_batch_markers: BatchMarkers = validator.active_batch_markers
     assert (