Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_equal.py b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_equal.py
--- a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_equal.py
+++ b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_equal.py
@@ -1,5 +1,3 @@
-from dateutil.parser import parse
-
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
     SqlAlchemyExecutionEngine,
diff --git a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_greater.py b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_greater.py
--- a/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_greater.py
+++ b/great_expectations/expectations/metrics/column_pair_map_metrics/column_pair_values_greater.py
@@ -38,9 +38,7 @@ def _pandas(cls, column_A, column_B, **kwargs):
 
         parse_strings_as_datetimes = kwargs.get("parse_strings_as_datetimes")
         if parse_strings_as_datetimes:
-            # noinspection PyPep8Naming
             temp_column_A = column_A.map(parse)
-            # noinspection PyPep8Naming
             temp_column_B = column_B.map(parse)
         else:
             temp_column_A = column_A
diff --git a/great_expectations/expectations/metrics/multicolumn_map_metrics/select_column_values_unique_within_record.py b/great_expectations/expectations/metrics/multicolumn_map_metrics/select_column_values_unique_within_record.py
--- a/great_expectations/expectations/metrics/multicolumn_map_metrics/select_column_values_unique_within_record.py
+++ b/great_expectations/expectations/metrics/multicolumn_map_metrics/select_column_values_unique_within_record.py
@@ -1,9 +1,17 @@
-from great_expectations.execution_engine import PandasExecutionEngine
+import logging
+
+from great_expectations.execution_engine import (
+    PandasExecutionEngine,
+    SqlAlchemyExecutionEngine,
+)
+from great_expectations.expectations.metrics.import_manager import sa
 from great_expectations.expectations.metrics.map_metric_provider import (
     MulticolumnMapMetricProvider,
     multicolumn_condition_partial,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class SelectColumnValuesUniqueWithinRecord(MulticolumnMapMetricProvider):
     condition_metric_name = "select_column_values.unique.within_record"
@@ -16,9 +24,37 @@ class SelectColumnValuesUniqueWithinRecord(MulticolumnMapMetricProvider):
         "ignore_row_if",
     )
 
-    # TODO: <Alex>ALEX -- temporarily only a Pandas implementation is provided (others to follow).</Alex>
+    # TODO: <Alex>ALEX -- temporarily only Pandas and SQL Alchemy implementations are provided (Spark to follow).</Alex>
     @multicolumn_condition_partial(engine=PandasExecutionEngine)
     def _pandas(cls, column_list, **kwargs):
         num_columns = len(column_list.columns)
         row_wise_cond = column_list.nunique(dropna=False, axis=1) >= num_columns
         return row_wise_cond
+
+    @multicolumn_condition_partial(engine=SqlAlchemyExecutionEngine)
+    def _sqlalchemy(cls, column_list, **kwargs):
+        """
+        The present approach relies on an inefficient query condition construction implementation, whose computational
+        cost is O(num_columns^2).  However, until a more efficient implementation compatible with SQLAlchemy is
+        available, this is the only feasible mechanism under the current architecture, where map metric providers must
+        return a condition.  Nevertheless, SQL query length limit is 1GB (sufficient for most practical scenarios).
+        """
+        num_columns = len(column_list)
+
+        # An arbitrary "num_columns" value used for issuing an explanatory message as a warning.
+        if num_columns > 100:
+            logger.warning(
+                f"""Batch data with {num_columns} columns is detected.  Computing the "{cls.condition_metric_name}" \
+metric for wide tables using SQLAlchemy leads to long WHERE clauses for the underlying database engine to process.
+"""
+            )
+
+        condition = sa.or_()
+        for idx_src in range(num_columns - 1):
+            for idx_dest in range(idx_src + 1, num_columns):
+                condition = sa.or_(
+                    condition, (column_list[idx_src] == column_list[idx_dest])
+                )
+
+        condition = sa.not_(condition)
+        return sa.case((condition, True), else_=False)


Test patch:
diff --git a/tests/expectations/metrics/test_core.py b/tests/expectations/metrics/test_core.py
--- a/tests/expectations/metrics/test_core.py
+++ b/tests/expectations/metrics/test_core.py
@@ -3032,3 +3032,208 @@ def test_map_select_column_values_unique_within_record_pd():
         {"a": 1.0, "b": 1.0, "c": 2.0},
         {"a": 4.0, "b": 4.0, "c": 4.0},
     ]
+
+
+def test_map_select_column_values_unique_within_record_sa(sa):
+    engine = build_sa_engine(
+        pd.DataFrame(
+            data={
+                "a": [1, 1, 8, 1, 4, None, None, 7],
+                "b": [1, 2, 2, 2, 4, None, None, 1],
+                "c": [2, 3, 7, 3, 4, None, 9, 0],
+            }
+        ),
+        sa,
+    )
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    # Save original metrics for testing unexpected results.
+    metrics_save: dict = copy.deepcopy(metrics)
+
+    metric_name: str = "select_column_values.unique.within_record"
+    condition_metric_name: str = f"{metric_name}.condition"
+    unexpected_count_metric_name: str = f"{metric_name}.unexpected_count"
+    unexpected_rows_metric_name: str = f"{metric_name}.unexpected_rows"
+    unexpected_values_metric_name: str = f"{metric_name}.unexpected_values"
+
+    condition_metric = MetricConfiguration(
+        metric_name=condition_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "all_values_are_missing",
+        },
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(condition_metric,),
+        metrics=metrics,
+    )
+    metrics.update(results)
+
+    unexpected_count_metric = MetricConfiguration(
+        metric_name=unexpected_count_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "all_values_are_missing",
+        },
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_count_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    # Condition metrics return "negative logic" series.
+    assert metrics[unexpected_count_metric.id] == 3
+
+    unexpected_rows_metric = MetricConfiguration(
+        metric_name=unexpected_rows_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "all_values_are_missing",
+        },
+        metric_value_kwargs={
+            "result_format": {"result_format": "SUMMARY", "partial_unexpected_count": 8}
+        },
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_rows_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    assert metrics[unexpected_rows_metric.id] == [
+        (1.0, 1.0, 2.0),
+        (4.0, 4.0, 4.0),
+        (None, None, 9.0),
+    ]
+
+    unexpected_values_metric = MetricConfiguration(
+        metric_name=unexpected_values_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "all_values_are_missing",
+        },
+        metric_value_kwargs={
+            "result_format": {"result_format": "SUMMARY", "partial_unexpected_count": 3}
+        },
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_values_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    assert len(metrics[unexpected_values_metric.id]) == 3
+
+    assert metrics[unexpected_values_metric.id] == [
+        {"a": 1.0, "b": 1.0, "c": 2.0},
+        {"a": 4.0, "b": 4.0, "c": 4.0},
+        {"a": None, "b": None, "c": 9.0},
+    ]
+
+    # Restore from saved original metrics in order to start fresh on testing for unexpected results.
+    metrics = copy.deepcopy(metrics_save)
+
+    condition_metric = MetricConfiguration(
+        metric_name=condition_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "any_value_is_missing",
+        },
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(condition_metric,),
+        metrics=metrics,
+    )
+    metrics.update(results)
+
+    unexpected_count_metric = MetricConfiguration(
+        metric_name=unexpected_count_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "any_value_is_missing",
+        },
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_count_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    # Condition metrics return "negative logic" series.
+    assert metrics[unexpected_count_metric.id] == 2
+
+    unexpected_rows_metric = MetricConfiguration(
+        metric_name=unexpected_rows_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "any_value_is_missing",
+        },
+        metric_value_kwargs={
+            "result_format": {"result_format": "SUMMARY", "partial_unexpected_count": 3}
+        },
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_rows_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    assert metrics[unexpected_rows_metric.id] == [(1.0, 1.0, 2.0), (4.0, 4.0, 4.0)]
+
+    unexpected_values_metric = MetricConfiguration(
+        metric_name=unexpected_values_metric_name,
+        metric_domain_kwargs={
+            "column_list": ["a", "b", "c"],
+            "ignore_row_if": "any_value_is_missing",
+        },
+        metric_value_kwargs={
+            "result_format": {"result_format": "SUMMARY", "partial_unexpected_count": 3}
+        },
+        metric_dependencies={
+            "unexpected_condition": condition_metric,
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(unexpected_values_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    assert len(metrics[unexpected_values_metric.id]) == 2
+    assert metrics[unexpected_values_metric.id] == [
+        {"a": 1.0, "b": 1.0, "c": 2.0},
+        {"a": 4.0, "b": 4.0, "c": 4.0},
+    ]