Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_in_set.py b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_in_set.py
--- a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_in_set.py
+++ b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_in_set.py
@@ -1,66 +1,42 @@
-import copy
-from typing import Any, Dict, Tuple
+# TODO: <Alex>ALEX -- Uncomment later (if needed for SparkDFExecutionEngine and/or SqlAlchemyExecutionEngine) or delete.</Alex>
+# import copy
+# from typing import Any, Dict, Tuple
 
 import numpy as np
 import pandas as pd
-from dateutil.parser import parse
 
-from great_expectations.execution_engine import (
-    PandasExecutionEngine,
-    SparkDFExecutionEngine,
-)
-from great_expectations.execution_engine.execution_engine import (
-    MetricDomainTypes,
-    MetricPartialFunctionTypes,
-)
-from great_expectations.expectations.metrics.import_manager import F, SQLContext
+# TODO: <Alex>ALEX -- Uncomment later (if needed for SparkDFExecutionEngine and/or SqlAlchemyExecutionEngine) or delete.</Alex>
+# from great_expectations.execution_engine import (
+#     SparkDFExecutionEngine,
+# )
+# from great_expectations.execution_engine.execution_engine import (
+#     MetricDomainTypes,
+#     MetricPartialFunctionTypes,
+# )
+# from great_expectations.expectations.metrics.import_manager import F, SQLContext
+from great_expectations.execution_engine import PandasExecutionEngine
 from great_expectations.expectations.metrics.map_metric_provider import (
-    MapMetricProvider,
+    ColumnPairMapMetricProvider,
+    column_pair_condition_partial,
 )
-from great_expectations.expectations.metrics.metric_provider import metric_partial
-from great_expectations.expectations.metrics.util import filter_pair_metric_nulls
 
 
-class ColumnPairValuesInSet(MapMetricProvider):
+class ColumnPairValuesInSet(ColumnPairMapMetricProvider):
     condition_metric_name = "column_pair_values.in_set"
-    condition_value_keys = (
-        "value_pairs_set",
+    condition_value_keys = ("value_pairs_set",)
+    condition_domain_keys = (
+        "batch_id",
+        "table",
+        "column_A",
+        "column_B",
         "ignore_row_if",
     )
-    condition_domain_keys = ("batch_id", "table", "column_A", "column_B")
-
-    @metric_partial(
-        engine=PandasExecutionEngine,
-        partial_fn_type=MetricPartialFunctionTypes.MAP_CONDITION_SERIES,
-        domain_type=MetricDomainTypes.COLUMN_PAIR,
-    )
-    def _pandas(
-        cls,
-        execution_engine: "PandasExecutionEngine",
-        metric_domain_kwargs: Dict,
-        metric_value_kwargs: Dict,
-        metrics: Dict[Tuple, Any],
-        runtime_configuration: Dict,
-    ):
 
-        value_pairs_set = metric_value_kwargs.get("value_pairs_set")
-        ignore_row_if = metric_value_kwargs.get("ignore_row_if")
-        if not ignore_row_if:
-            ignore_row_if = "both_values_are_missing"
-
-        (
-            df,
-            compute_domain_kwargs,
-            accessor_domain_kwargs,
-        ) = execution_engine.get_compute_domain(
-            metric_domain_kwargs, domain_type=MetricDomainTypes.COLUMN_PAIR
-        )
-
-        column_A, column_B = filter_pair_metric_nulls(
-            df[metric_domain_kwargs["column_A"]],
-            df[metric_domain_kwargs["column_B"]],
-            ignore_row_if=ignore_row_if,
-        )
+    # TODO: <Alex>ALEX -- temporarily only a Pandas implementation is provided (others to follow).</Alex>
+    # noinspection PyPep8Naming
+    @column_pair_condition_partial(engine=PandasExecutionEngine)
+    def _pandas(cls, column_A, column_B, **kwargs):
+        value_pairs_set = kwargs.get("value_pairs_set")
 
         if value_pairs_set is None:
             # vacuously true
@@ -83,63 +59,64 @@ def _pandas(
 
             results.append((a, b) in value_pairs_set)
 
-        return pd.Series(results), compute_domain_kwargs, accessor_domain_kwargs
-
-    @metric_partial(
-        engine=SparkDFExecutionEngine,
-        partial_fn_type=MetricPartialFunctionTypes.WINDOW_CONDITION_FN,
-        domain_type=MetricDomainTypes.COLUMN_PAIR,
-    )
-    def _spark(
-        cls,
-        execution_engine: "SparkDFExecutionEngine",
-        metric_domain_kwargs: Dict,
-        metric_value_kwargs: Dict,
-        metrics: Dict[Tuple, Any],
-        runtime_configuration: Dict,
-    ):
-        ignore_row_if = metric_value_kwargs["ignore_row_if"]
-        compute_domain_kwargs = copy.deepcopy(metric_domain_kwargs)
-
-        if ignore_row_if == "both_values_are_missing":
-            compute_domain_kwargs["row_condition"] = (
-                F.col(metric_domain_kwargs["column_A"]).isNotNull()
-                & F.col(metric_domain_kwargs["column_B"]).isNotNull()
-            )
-            compute_domain_kwargs["condition_parser"] = "spark"
-        elif ignore_row_if == "either_value_is_missing":
-            compute_domain_kwargs["row_condition"] = (
-                F.col(metric_domain_kwargs["column_A"]).isNotNull()
-                | F.col(metric_domain_kwargs["column_B"]).isNotNull()
-            )
-            compute_domain_kwargs["condition_parser"] = "spark"
-
-        (
-            df,
-            compute_domain_kwargs,
-            accessor_domain_kwargs,
-        ) = execution_engine.get_compute_domain(
-            compute_domain_kwargs, domain_type=MetricDomainTypes.COLUMN_PAIR
-        )
-
-        df = df.withColumn(
-            "combined",
-            F.array(
-                F.col(metric_domain_kwargs["column_A"]),
-                F.col(metric_domain_kwargs["column_A"]),
-            ),
-        )
-
-        value_set_df = (
-            SQLContext(df._sc)
-            .createDataFrame(metric_value_kwargs["value_pairs_set"], ["col_A", "col_B"])
-            .select(F.array("col_A", "col_B").alias("set_AB"))
-        )
-
-        df = df.join(
-            value_set_df, df["combined"] == value_set_df["set_AB"], "left"
-        ).withColumn(
-            "__success",
-            F.when(F.col("set_AB").isNull(), F.lit(False)).otherwise(F.lit(True)),
-        )
-        return df["__success"], compute_domain_kwargs, accessor_domain_kwargs
+        return pd.Series(results)
+
+    # # TODO: <Alex>ALEX -- Note: The Spark implementation below is a temporary placeholder.</Alex>
+    # @metric_partial(
+    #     engine=SparkDFExecutionEngine,
+    #     partial_fn_type=MetricPartialFunctionTypes.WINDOW_CONDITION_FN,
+    #     domain_type=MetricDomainTypes.COLUMN_PAIR,
+    # )
+    # def _spark(
+    #     cls,
+    #     execution_engine: "SparkDFExecutionEngine",
+    #     metric_domain_kwargs: Dict,
+    #     metric_value_kwargs: Dict,
+    #     metrics: Dict[Tuple, Any],
+    #     runtime_configuration: Dict,
+    # ):
+    #     ignore_row_if = metric_value_kwargs["ignore_row_if"]
+    #     compute_domain_kwargs = copy.deepcopy(metric_domain_kwargs)
+    #
+    #     if ignore_row_if == "both_values_are_missing":
+    #         compute_domain_kwargs["row_condition"] = (
+    #             F.col(metric_domain_kwargs["column_A"]).isNotNull()
+    #             & F.col(metric_domain_kwargs["column_B"]).isNotNull()
+    #         )
+    #         compute_domain_kwargs["condition_parser"] = "spark"
+    #     elif ignore_row_if == "either_value_is_missing":
+    #         compute_domain_kwargs["row_condition"] = (
+    #             F.col(metric_domain_kwargs["column_A"]).isNotNull()
+    #             | F.col(metric_domain_kwargs["column_B"]).isNotNull()
+    #         )
+    #         compute_domain_kwargs["condition_parser"] = "spark"
+    #
+    #     (
+    #         df,
+    #         compute_domain_kwargs,
+    #         accessor_domain_kwargs,
+    #     ) = execution_engine.get_compute_domain(
+    #         compute_domain_kwargs, domain_type=MetricDomainTypes.COLUMN_PAIR
+    #     )
+    #
+    #     df = df.withColumn(
+    #         "combined",
+    #         F.array(
+    #             F.col(metric_domain_kwargs["column_A"]),
+    #             F.col(metric_domain_kwargs["column_A"]),
+    #         ),
+    #     )
+    #
+    #     value_set_df = (
+    #         SQLContext(df._sc)
+    #         .createDataFrame(metric_value_kwargs["value_pairs_set"], ["col_A", "col_B"])
+    #         .select(F.array("col_A", "col_B").alias("set_AB"))
+    #     )
+    #
+    #     df = df.join(
+    #         value_set_df, df["combined"] == value_set_df["set_AB"], "left"
+    #     ).withColumn(
+    #         "__success",
+    #         F.when(F.col("set_AB").isNull(), F.lit(False)).otherwise(F.lit(True)),
+    #     )
+    #     return df["__success"], compute_domain_kwargs, accessor_domain_kwargs


Test patch:
diff --git a/tests/expectations/metrics/test_core.py b/tests/expectations/metrics/test_core.py
--- a/tests/expectations/metrics/test_core.py
+++ b/tests/expectations/metrics/test_core.py
@@ -386,7 +386,7 @@ def test_map_of_type_sa(sa):
     desired_metric = MetricConfiguration(
         metric_name="table.column_types",
         metric_domain_kwargs={},
-        metric_value_kwargs={},
+        metric_value_kwargs=dict(),
     )
 
     results = engine.resolve_metrics(metrics_to_resolve=(desired_metric,))
@@ -1149,19 +1149,39 @@ def test_column_pairs_greater_metric_pd():
 def test_column_pairs_in_set_metric_pd():
     df = pd.DataFrame({"a": [10, 3, 4, None, 3, None], "b": [1, 2, 3, None, 3, 5]})
     engine = PandasExecutionEngine(batch_data_dict={"my_id": df})
-    desired_metric = MetricConfiguration(
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    condition_metric = MetricConfiguration(
         metric_name="column_pair_values.in_set.condition",
-        metric_domain_kwargs={"column_A": "a", "column_B": "b"},
+        metric_domain_kwargs={
+            "column_A": "a",
+            "column_B": "b",
+            "ignore_row_if": "either_value_is_missing",
+        },
         metric_value_kwargs={
             "value_pairs_set": [(2, 1), (3, 2), (4, 3), (3, 3)],
-            "ignore_row_if": "either_value_is_missing",
+        },
+        metric_dependencies={
+            "table.columns": table_columns_metric,
         },
     )
-    results = engine.resolve_metrics(metrics_to_resolve=(desired_metric,))
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(condition_metric,),
+        metrics=metrics,
+    )
+    metrics.update(results)
+
     assert (
-        results[desired_metric.id][0]
+        results[condition_metric.id][0]
         .reset_index(drop=True)
-        .equals(pd.Series([False, True, True, True]))
+        .equals(pd.Series([True, False, False, False]))
     )