Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/rule_based_profiler/attributed_resolved_metrics.py b/great_expectations/rule_based_profiler/attributed_resolved_metrics.py
--- a/great_expectations/rule_based_profiler/attributed_resolved_metrics.py
+++ b/great_expectations/rule_based_profiler/attributed_resolved_metrics.py
@@ -1,3 +1,4 @@
+import logging
 from dataclasses import asdict, dataclass
 from typing import Any, Dict, Iterator, List, Optional, Sized
 
@@ -10,8 +11,29 @@
 )
 from great_expectations.types import SerializableDictDot
 from great_expectations.types.attributes import Attributes
+from great_expectations.util import deep_filter_properties_iterable
 from great_expectations.validator.computed_metric import MetricValue
 
+logger = logging.getLogger(__name__)
+
+try:
+    import sqlalchemy as sa
+except ImportError:
+    logger.debug("No SqlAlchemy module available.")
+    sa = None
+
+try:
+    from sqlalchemy.engine import Row as sqlalchemy_engine_Row
+except ImportError:
+    logger.debug("No SqlAlchemy.engine module available.")
+    sqlalchemy_engine_Row = None
+
+try:
+    from pyspark.sql import Row as pyspark_sql_Row
+except ImportError:
+    logger.debug("No spark SQLContext available.")
+    pyspark_sql_Row = None
+
 
 def _condition_metric_values(metric_values: MetricValues) -> MetricValues:
     def _detect_illegal_array_type_or_shape(values: MetricValues) -> bool:
@@ -20,7 +42,18 @@ def _detect_illegal_array_type_or_shape(values: MetricValues) -> bool:
             return True
 
         # Pandas "DataFrame" and "Series" are illegal as candidates for conversion into "numpy.ndarray" type.
-        if isinstance(values, (pd.DataFrame, pd.Series, set)):
+        if isinstance(
+            values,
+            deep_filter_properties_iterable(
+                properties=(
+                    pd.DataFrame,
+                    pd.Series,
+                    sqlalchemy_engine_Row,
+                    pyspark_sql_Row,
+                    set,
+                )
+            ),
+        ):
             return True
 
         if isinstance(values, (list, tuple)):
@@ -31,7 +64,18 @@ def _detect_illegal_array_type_or_shape(values: MetricValues) -> bool:
                     return True
 
                 # Pandas "DataFrame" and "Series" are illegal as candidates for conversion into "numpy.ndarray" type.
-                if isinstance(value, (pd.DataFrame, pd.Series, set)):
+                if isinstance(
+                    value,
+                    deep_filter_properties_iterable(
+                        properties=(
+                            pd.DataFrame,
+                            pd.Series,
+                            sqlalchemy_engine_Row,
+                            pyspark_sql_Row,
+                            set,
+                        )
+                    ),
+                ):
                     return True
 
                 # Components of different lengths cannot be packaged into "numpy.ndarray" type (due to undefined shape).
diff --git a/great_expectations/rule_based_profiler/parameter_container.py b/great_expectations/rule_based_profiler/parameter_container.py
--- a/great_expectations/rule_based_profiler/parameter_container.py
+++ b/great_expectations/rule_based_profiler/parameter_container.py
@@ -230,13 +230,13 @@ def to_json_dict(self) -> dict:
 def deep_convert_properties_iterable_to_parameter_node(
     source: Union[T, dict]
 ) -> Union[T, ParameterNode]:
-    if isinstance(source, dict):
+    if type(source) == dict:
         return _deep_convert_properties_iterable_to_parameter_node(
             source=ParameterNode(source)
         )
 
     # Must allow for non-dictionary source types, since their internal nested structures may contain dictionaries.
-    if isinstance(source, (list, set, tuple)):
+    if type(source) in (list, set, tuple):
         data_type: type = type(source)
 
         element: Any
@@ -254,11 +254,11 @@ def _deep_convert_properties_iterable_to_parameter_node(source: dict) -> Paramet
     key: str
     value: Any
     for key, value in source.items():
-        if isinstance(value, dict):
+        if type(value) == dict:
             source[key] = _deep_convert_properties_iterable_to_parameter_node(
                 source=value
             )
-        elif isinstance(value, (list, set, tuple)):
+        elif type(value) in (list, set, tuple):
             data_type: type = type(value)
 
             element: Any


Test patch:
diff --git a/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py b/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
--- a/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
+++ b/tests/expectations/core/test_expect_column_value_z_scores_to_be_less_than.py
@@ -1,14 +1,14 @@
+from typing import Optional, cast
+
 import pandas as pd
 import pytest
 
+from great_expectations import DataContext
 from great_expectations.core.expectation_validation_result import (
     ExpectationValidationResult,
 )
-from great_expectations.self_check.util import (
-    build_pandas_validator_with_data,
-    build_sa_validator_with_data,
-    build_spark_validator_with_data,
-)
+from great_expectations.self_check.util import get_test_validator_with_data
+from great_expectations.util import build_in_memory_runtime_context
 
 
 @pytest.fixture
@@ -49,7 +49,14 @@ def test_pandas_expect_column_value_z_scores_to_be_less_than_impl(
 ):
     df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
 
-    validator = build_pandas_validator_with_data(df)
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="pandas",
+        data=df,
+        context=context,
+    )
 
     result = validator.expect_column_value_z_scores_to_be_less_than(
         column="a", mostly=0.9, threshold=4, double_sided=True
@@ -66,7 +73,14 @@ def test_sa_expect_column_value_z_scores_to_be_less_than_impl(
 
     df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
 
-    validator = build_sa_validator_with_data(df=df, sa_engine_name="postgresql")
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="postgresql",
+        data=df,
+        context=context,
+    )
 
     result = validator.expect_column_value_z_scores_to_be_less_than(
         column="a", mostly=0.9, threshold=4, double_sided=True
@@ -75,12 +89,20 @@ def test_sa_expect_column_value_z_scores_to_be_less_than_impl(
     assert result == z_score_validation_result
 
 
+# noinspection PyUnusedLocal
 def test_spark_expect_column_value_z_scores_to_be_less_than_impl(
     spark_session, basic_spark_df_execution_engine, z_score_validation_result
 ):
     df = pd.DataFrame({"a": [1, 5, 22, 3, 5, 10]})
 
-    validator = build_spark_validator_with_data(df, spark_session)
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="spark",
+        data=df,
+        context=context,
+    )
 
     result = validator.expect_column_value_z_scores_to_be_less_than(
         column="a", mostly=0.9, threshold=4, double_sided=True
diff --git a/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py b/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
--- a/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
+++ b/tests/expectations/core/test_expect_column_values_to_be_in_type_list.py
@@ -1,14 +1,18 @@
+from typing import Optional, cast
+
 import pandas as pd
 import pytest
 
+from great_expectations import DataContext
 from great_expectations.core.expectation_validation_result import (
     ExpectationValidationResult,
 )
 from great_expectations.self_check.util import (
     build_pandas_validator_with_data,
     build_sa_validator_with_data,
+    get_test_validator_with_data,
 )
-from great_expectations.util import is_library_loadable
+from great_expectations.util import build_in_memory_runtime_context, is_library_loadable
 
 
 @pytest.mark.skipif(
@@ -112,7 +116,15 @@ def test_expect_column_values_to_be_in_type_list_nullable_int():
         pytest.skip("Prior to 0.24, Pandas did not have `Int32Dtype` or related.")
 
     df = pd.DataFrame({"col": pd.Series([1, 2, None], dtype=pd.Int32Dtype())})
-    validator = build_pandas_validator_with_data(df)
+
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="pandas",
+        data=df,
+        context=context,
+    )
 
     result = validator.expect_column_values_to_be_in_type_list(
         "col", type_list=["Int32Dtype"]
diff --git a/tests/expectations/core/test_expect_queried_column_value_frequency_to_meet_threshold.py b/tests/expectations/core/test_expect_queried_column_value_frequency_to_meet_threshold.py
--- a/tests/expectations/core/test_expect_queried_column_value_frequency_to_meet_threshold.py
+++ b/tests/expectations/core/test_expect_queried_column_value_frequency_to_meet_threshold.py
@@ -1,3 +1,5 @@
+from typing import Optional, cast
+
 import pandas as pd
 import pytest
 
@@ -7,7 +9,11 @@
 )
 from great_expectations.core.batch import BatchRequest, RuntimeBatchRequest
 from great_expectations.data_context import DataContext
-from great_expectations.self_check.util import build_spark_validator_with_data
+from great_expectations.self_check.util import (
+    build_spark_validator_with_data,
+    get_test_validator_with_data,
+)
+from great_expectations.util import build_in_memory_runtime_context
 from great_expectations.validator.validator import (
     ExpectationValidationResult,
     Validator,
@@ -165,12 +171,13 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_override_query_
     )
 
 
+# noinspection PyUnusedLocal
 @pytest.mark.parametrize(
     "success,observed,row_condition,warns",
     [
         (True, 0.6481340441736482, None, False),
         (False, 0.4791666666666667, 'col("Age")<18', True),
-        (True, 0.6393939393939394, 'col("Age")>17', True),
+        (True, 0.6614626129827444, 'col("Age")>17', True),
     ],
 )
 def test_expect_queried_column_value_frequency_to_meet_threshold_spark(
@@ -183,7 +190,15 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_spark(
     titanic_df,
 ):
     df: pd.DataFrame = titanic_df
-    validator: Validator = build_spark_validator_with_data(df, spark_session)
+
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="spark",
+        data=df,
+        context=context,
+    )
 
     if warns:
         with pytest.warns(UserWarning):
@@ -211,6 +226,7 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_spark(
     )
 
 
+# noinspection PyUnusedLocal
 @pytest.mark.parametrize(
     "query,success,observed,row_condition,warns",
     [
@@ -242,7 +258,14 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_override_query_
 ):
     df: pd.DataFrame = titanic_df
 
-    validator: Validator = build_spark_validator_with_data(df, spark_session)
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="spark",
+        data=df,
+        context=context,
+    )
 
     if warns:
         with pytest.warns(UserWarning):
diff --git a/tests/expectations/core/test_expect_queried_table_row_count_to_be.py b/tests/expectations/core/test_expect_queried_table_row_count_to_be.py
--- a/tests/expectations/core/test_expect_queried_table_row_count_to_be.py
+++ b/tests/expectations/core/test_expect_queried_table_row_count_to_be.py
@@ -1,3 +1,5 @@
+from typing import Optional, cast
+
 import pandas as pd
 import pytest
 
@@ -7,7 +9,8 @@
 )
 from great_expectations.core.batch import BatchRequest, RuntimeBatchRequest
 from great_expectations.data_context import DataContext
-from great_expectations.self_check.util import build_spark_validator_with_data
+from great_expectations.self_check.util import get_test_validator_with_data
+from great_expectations.util import build_in_memory_runtime_context
 from great_expectations.validator.validator import (
     ExpectationValidationResult,
     Validator,
@@ -114,6 +117,7 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_override_query_
     )
 
 
+# noinspection PyUnusedLocal
 @pytest.mark.parametrize(
     "success,value,observed,row_condition",
     [
@@ -132,7 +136,14 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_spark(
 ):
     df: pd.DataFrame = titanic_df
 
-    validator: Validator = build_spark_validator_with_data(df, spark_session)
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="spark",
+        data=df,
+        context=context,
+    )
 
     result: ExpectationValidationResult = (
         validator.expect_queried_table_row_count_to_be(
@@ -147,6 +158,7 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_spark(
     )
 
 
+# noinspection PyUnusedLocal
 @pytest.mark.parametrize(
     "success,query,value,observed,row_condition",
     [
@@ -171,7 +183,14 @@ def test_expect_queried_column_value_frequency_to_meet_threshold_override_query_
 ):
     df: pd.DataFrame = titanic_df
 
-    validator: Validator = build_spark_validator_with_data(df, spark_session)
+    context: Optional[DataContext] = cast(
+        DataContext, build_in_memory_runtime_context()
+    )
+    validator = get_test_validator_with_data(
+        execution_engine="spark",
+        data=df,
+        context=context,
+    )
 
     result: ExpectationValidationResult = (
         validator.expect_queried_table_row_count_to_be(
diff --git a/tests/rule_based_profiler/parameter_builder/test_parameter_computations.py b/tests/rule_based_profiler/parameter_builder/test_parameter_computations.py
--- a/tests/rule_based_profiler/parameter_builder/test_parameter_computations.py
+++ b/tests/rule_based_profiler/parameter_builder/test_parameter_computations.py
@@ -22,7 +22,6 @@
 # Allowable tolerance for how closely a bootstrap method approximates the sample
 from great_expectations.rule_based_profiler.parameter_builder import (
     MetricMultiBatchParameterBuilder,
-    ParameterBuilder,
 )
 from great_expectations.rule_based_profiler.parameter_container import (
     DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME,
@@ -212,7 +211,7 @@ def test_sanitize_metric_computation(metric_name: str, metric_values_by_batch_id
     replace_nan_with_zero: bool = True
     reduce_scalar_metric: bool = True
 
-    metric_multi_batch_parameter_builder: ParameterBuilder = (
+    metric_multi_batch_parameter_builder: MetricMultiBatchParameterBuilder = (
         MetricMultiBatchParameterBuilder(
             name="my_parameter_builder",
             metric_name=metric_name,
diff --git a/tests/test_definitions/test_expectations_v3_api.py b/tests/test_definitions/test_expectations_v3_api.py
--- a/tests/test_definitions/test_expectations_v3_api.py
+++ b/tests/test_definitions/test_expectations_v3_api.py
@@ -1,10 +1,12 @@
 import glob
 import json
 import os
+from typing import cast
 
 import pandas as pd
 import pytest
 
+from great_expectations import DataContext
 from great_expectations.execution_engine.pandas_batch_data import PandasBatchData
 from great_expectations.execution_engine.sparkdf_batch_data import SparkDFBatchData
 from great_expectations.execution_engine.sqlalchemy_batch_data import (
@@ -80,7 +82,10 @@ def pytest_generate_tests(metafunc):
                                         dataset.get("schemas"),
                                         table_name=dataset.get("dataset_name"),
                                         sqlite_db_path=sqlite_db_path,
-                                        context=build_in_memory_runtime_context(),
+                                        context=cast(
+                                            DataContext,
+                                            build_in_memory_runtime_context(),
+                                        ),
                                     )
                                 )
                             validator_with_data = datasets[0]
@@ -90,7 +95,9 @@ def pytest_generate_tests(metafunc):
                                 c,
                                 d["data"],
                                 schemas=schemas,
-                                context=build_in_memory_runtime_context(),
+                                context=cast(
+                                    DataContext, build_in_memory_runtime_context()
+                                ),
                             )
 
                     for test in d["tests"]: