Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_pair_values_to_have_diff.py b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_pair_values_to_have_diff.py
new file mode 100644
--- /dev/null
+++ b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_pair_values_to_have_diff.py
@@ -0,0 +1,170 @@
+"""
+This is an example of a Custom QueryExpectation.
+For detailed information on QueryExpectations, please see:
+    https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_query_expectations
+"""
+
+from typing import Optional, Union
+
+from great_expectations.core.expectation_configuration import ExpectationConfiguration
+from great_expectations.exceptions.exceptions import (
+    InvalidExpectationConfigurationError,
+)
+from great_expectations.execution_engine import ExecutionEngine
+from great_expectations.expectations.expectation import (
+    ExpectationValidationResult,
+    QueryExpectation,
+)
+from great_expectations.expectations.metrics.import_manager import (
+    pyspark_sql_Row,
+    sqlalchemy_engine_Row,
+)
+
+
+class ExpectQueriedColumnPairValuesToHaveDiff(QueryExpectation):
+    """Expect the frequency of occurrences of a specified value in a queried column to be at least <mostly> percent of values in that column."""
+
+    metric_dependencies = ("query.column_pair",)
+
+    query = """
+            SELECT {column_A} - {column_B} as diff
+            FROM {active_batch}
+            """
+
+    success_keys = ("column_A", "column_B", "diff", "mostly", "query", "strict")
+
+    domain_keys = ("batch_id", "row_condition", "condition_parser")
+
+    default_kwarg_values = {
+        "result_format": "BASIC",
+        "include_config": True,
+        "catch_exceptions": False,
+        "meta": None,
+        "column_A": None,
+        "column_B": None,
+        "diff": 0,
+        "mostly": 1,
+        "strict": True,
+        "query": query,
+    }
+
+    def validate_configuration(
+        self, configuration: Optional[ExpectationConfiguration]
+    ) -> None:
+        super().validate_configuration(configuration)
+        diff = configuration["kwargs"].get("diff")
+        mostly = configuration["kwargs"].get("mostly")
+
+        try:
+            assert diff is not None, "'diff' must be specified"
+            assert isinstance(diff, (int, float)), "`diff` must be a valid float or int"
+            assert (
+                isinstance(mostly, (int, float)) and 0 < mostly <= 1
+            ), "'mostly' must be 1, a float between 0 and 1"
+        except AssertionError as e:
+            raise InvalidExpectationConfigurationError(str(e))
+
+    def _validate(
+        self,
+        configuration: ExpectationConfiguration,
+        metrics: dict,
+        runtime_configuration: dict = None,
+        execution_engine: ExecutionEngine = None,
+    ) -> Union[ExpectationValidationResult, dict]:
+        diff: Union[float, int] = configuration["kwargs"].get("diff")
+        mostly: str = configuration["kwargs"].get("mostly")
+        query_result: Union[sqlalchemy_engine_Row, pyspark_sql_Row] = metrics.get(
+            "query.column_pair"
+        )
+
+        success = (
+            sum([(abs(x[0]) == diff) for x in query_result]) / len(query_result)
+        ) >= mostly
+
+        return {
+            "success": success,
+            "result": {"observed_value": [x[0] for x in query_result]},
+        }
+
+    examples = [
+        {
+            "data": {
+                "col1": [1, 2, 2, 3, 4],
+                "col2": [2, 3, 3, 4, 2],
+            },
+            "tests": [
+                {
+                    "title": "basic_positive_test",
+                    "exact_match_out": False,
+                    "include_in_gallery": True,
+                    "in": {
+                        "column_A": "col1",
+                        "column_B": "col2",
+                        "diff": 1,
+                        "mostly": 0.6,
+                    },
+                    "out": {"success": True},
+                    "only_for": ["sqlite", "spark"],
+                },
+                {
+                    "title": "basic_negative_test",
+                    "exact_match_out": False,
+                    "include_in_gallery": True,
+                    "in": {
+                        "column_A": "col1",
+                        "column_B": "col2",
+                        "diff": 2,
+                        "mostly": 0.4,
+                    },
+                    "out": {"success": False},
+                    "only_for": ["sqlite", "spark"],
+                },
+            ],
+        },
+        {
+            "data": {
+                "col1": [1, 2, 2, 3, 4],
+                "col2": [2, 3, 3, 4, 2],
+                "col3": [True, False, "a", 3, 5],
+                "col4": [0.1, 22.3, 4, 4.2, 6],
+            },
+            "tests": [
+                {
+                    "title": "mixed_types_positive_test",
+                    "exact_match_out": False,
+                    "include_in_gallery": True,
+                    "in": {
+                        "column_A": "col1",
+                        "column_B": "col3",
+                        "diff": 2,
+                        "mostly": 0.4,
+                    },
+                    "out": {"success": True},
+                    "only_for": ["sqlite"],
+                },
+                {
+                    "title": "floating_point_negative_test",
+                    "exact_match_out": False,
+                    "include_in_gallery": True,
+                    "in": {
+                        "column_A": "col2",
+                        "column_B": "col4",
+                        "diff": 1,
+                        "mostly": 0.4,
+                    },
+                    "out": {"success": False},
+                    "only_for": ["sqlite"],
+                },
+            ],
+        },
+    ]
+
+    # This dictionary contains metadata for display in the public gallery
+    library_metadata = {
+        "tags": ["query-based"],
+        "contributors": ["@joegargery"],
+    }
+
+
+if __name__ == "__main__":
+    ExpectQueriedColumnPairValuesToHaveDiff().print_diagnostic_checklist()
diff --git a/great_expectations/expectations/metrics/query_metrics/__init__.py b/great_expectations/expectations/metrics/query_metrics/__init__.py
--- a/great_expectations/expectations/metrics/query_metrics/__init__.py
+++ b/great_expectations/expectations/metrics/query_metrics/__init__.py
@@ -1,2 +1,3 @@
 from .query_column import QueryColumn
+from .query_column_pair import QueryColumnPair
 from .query_table import QueryTable
diff --git a/great_expectations/expectations/metrics/query_metrics/query_column_pair.py b/great_expectations/expectations/metrics/query_metrics/query_column_pair.py
new file mode 100644
--- /dev/null
+++ b/great_expectations/expectations/metrics/query_metrics/query_column_pair.py
@@ -0,0 +1,107 @@
+from typing import Any, Dict, List, Optional, Union
+
+from great_expectations.core.metric_domain_types import MetricDomainTypes
+from great_expectations.execution_engine import (
+    SparkDFExecutionEngine,
+    SqlAlchemyExecutionEngine,
+)
+from great_expectations.expectations.metrics.import_manager import (
+    pyspark_sql_DataFrame,
+    pyspark_sql_Row,
+    pyspark_sql_SparkSession,
+    sa,
+    sqlalchemy_engine_Engine,
+    sqlalchemy_engine_Row,
+)
+from great_expectations.expectations.metrics.metric_provider import metric_value
+from great_expectations.expectations.metrics.query_metric_provider import (
+    QueryMetricProvider,
+)
+from great_expectations.util import get_sqlalchemy_subquery_type
+
+
+class QueryColumnPair(QueryMetricProvider):
+    metric_name = "query.column_pair"
+    value_keys = (
+        "column_A",
+        "column_B",
+        "query",
+    )
+
+    @metric_value(engine=SqlAlchemyExecutionEngine)
+    def _sqlalchemy(
+        cls,
+        execution_engine: SqlAlchemyExecutionEngine,
+        metric_domain_kwargs: dict,
+        metric_value_kwargs: dict,
+        metrics: Dict[str, Any],
+        runtime_configuration: dict,
+    ) -> List[sqlalchemy_engine_Row]:
+        query: Optional[str] = metric_value_kwargs.get(
+            "query"
+        ) or cls.default_kwarg_values.get("query")
+
+        selectable: Union[sa.sql.Selectable, str]
+        selectable, _, _ = execution_engine.get_compute_domain(
+            metric_domain_kwargs, domain_type=MetricDomainTypes.TABLE
+        )
+
+        column_A: str = metric_value_kwargs.get("column_A")
+        column_B: str = metric_value_kwargs.get("column_B")
+        if isinstance(selectable, sa.Table):
+            query = query.format(
+                column_A=column_A, column_B=column_B, active_batch=selectable
+            )
+        elif isinstance(
+            selectable, get_sqlalchemy_subquery_type()
+        ):  # Specifying a runtime query in a RuntimeBatchRequest returns the active bacth as a Subquery; sectioning the active batch off w/ parentheses ensures flow of operations doesn't break
+            query = query.format(
+                column_A=column_A, column_B=column_B, active_batch=f"({selectable})"
+            )
+        elif isinstance(
+            selectable, sa.sql.Select
+        ):  # Specifying a row_condition returns the active batch as a Select object, requiring compilation & aliasing when formatting the parameterized query
+            query = query.format(
+                column_A=column_A,
+                column_B=column_B,
+                active_batch=f'({selectable.compile(compile_kwargs={"literal_binds": True})}) AS subselect',
+            )
+        else:
+            query = query.format(
+                column_A=column_A, column_B=column_B, active_batch=f"({selectable})"
+            )
+
+        engine: sqlalchemy_engine_Engine = execution_engine.engine
+        result: List[sqlalchemy_engine_Row] = engine.execute(sa.text(query)).fetchall()
+
+        return result
+
+    @metric_value(engine=SparkDFExecutionEngine)
+    def _spark(
+        cls,
+        execution_engine: SparkDFExecutionEngine,
+        metric_domain_kwargs: dict,
+        metric_value_kwargs: dict,
+        metrics: Dict[str, Any],
+        runtime_configuration: dict,
+    ) -> List[pyspark_sql_Row]:
+        query: Optional[str] = metric_value_kwargs.get(
+            "query"
+        ) or cls.default_kwarg_values.get("query")
+
+        df: pyspark_sql_DataFrame
+        df, _, _ = execution_engine.get_compute_domain(
+            metric_domain_kwargs, domain_type=MetricDomainTypes.TABLE
+        )
+
+        df.createOrReplaceTempView("tmp_view")
+        column_A: str = metric_value_kwargs.get("column_A")
+        column_B: str = metric_value_kwargs.get("column_B")
+        query = query.format(
+            column_A=column_A, column_B=column_B, active_batch="tmp_view"
+        )
+
+        engine: pyspark_sql_SparkSession = execution_engine.spark
+        result: List[pyspark_sql_Row] = engine.sql(query).collect()
+
+        return result


Test patch: