Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/execution_engine/execution_engine.py b/great_expectations/execution_engine/execution_engine.py
--- a/great_expectations/execution_engine/execution_engine.py
+++ b/great_expectations/execution_engine/execution_engine.py
@@ -20,10 +20,6 @@
 import great_expectations.exceptions as gx_exceptions
 from great_expectations.core.batch_manager import BatchManager
 from great_expectations.core.metric_domain_types import MetricDomainTypes
-from great_expectations.core.metric_function_types import (
-    MetricFunctionTypes,
-    MetricPartialFunctionTypes,
-)
 from great_expectations.core.util import (
     AzureUrl,
     DBFSPath,
@@ -42,6 +38,10 @@
 from great_expectations.validator.metric_configuration import MetricConfiguration
 
 if TYPE_CHECKING:
+    # noinspection PyPep8Naming
+    import pyspark.sql.functions as F
+    import sqlalchemy as sa
+
     from great_expectations.core.batch import (
         BatchData,
         BatchDataType,
@@ -82,7 +82,7 @@ class MetricComputationConfiguration(DictDot):
     """
 
     metric_configuration: MetricConfiguration
-    metric_fn: Any
+    metric_fn: sa.func | F
     metric_provider_kwargs: dict
     compute_domain_kwargs: Optional[dict] = None
     accessor_domain_kwargs: Optional[dict] = None
@@ -441,6 +441,8 @@ def _build_direct_and_bundled_metric_computation_configurations(
         Directly-computable "MetricConfiguration" must have non-NULL metric function ("metric_fn").  Aggregate metrics
         have NULL metric function, but non-NULL partial metric function ("metric_partial_fn"); aggregates are bundled.
 
+        See documentation in "MetricProvider._register_metric_functions()" for in-depth description of this mechanism.
+
         Args:
             metrics_to_resolve: the metrics to evaluate
             metrics: already-computed metrics currently available to the engine
@@ -465,7 +467,8 @@ def _build_direct_and_bundled_metric_computation_configurations(
             str, Union[MetricValue, Tuple[Any, dict, dict]]
         ]
         metric_class: MetricProvider
-        metric_fn: Any
+        metric_fn: Union[Callable, None]
+        metric_aggregate_fn: sa.func | F
         metric_provider_kwargs: dict
         compute_domain_kwargs: dict
         accessor_domain_kwargs: dict
@@ -491,10 +494,10 @@ def _build_direct_and_bundled_metric_computation_configurations(
             if metric_fn is None:
                 try:
                     (
-                        metric_fn,
+                        metric_aggregate_fn,
                         compute_domain_kwargs,
                         accessor_domain_kwargs,
-                    ) = resolved_metric_dependencies_by_metric_name.pop(  # type: ignore[misc,assignment]
+                    ) = resolved_metric_dependencies_by_metric_name.pop(
                         "metric_partial_fn"
                     )
                 except KeyError as e:
@@ -505,34 +508,13 @@ def _build_direct_and_bundled_metric_computation_configurations(
                 metric_fn_bundle_configurations.append(
                     MetricComputationConfiguration(
                         metric_configuration=metric_to_resolve,
-                        metric_fn=metric_fn,
+                        metric_fn=metric_aggregate_fn,
                         metric_provider_kwargs=metric_provider_kwargs,
                         compute_domain_kwargs=compute_domain_kwargs,
                         accessor_domain_kwargs=accessor_domain_kwargs,
                     )
                 )
             else:
-                metric_fn_type: MetricFunctionTypes = getattr(
-                    metric_fn, "metric_fn_type", MetricFunctionTypes.VALUE
-                )
-                if isinstance(
-                    metric_fn_type, (MetricFunctionTypes, MetricPartialFunctionTypes)
-                ) and metric_fn_type not in [
-                    MetricPartialFunctionTypes.MAP_FN,
-                    MetricPartialFunctionTypes.MAP_SERIES,
-                    MetricPartialFunctionTypes.WINDOW_FN,
-                    MetricPartialFunctionTypes.MAP_CONDITION_FN,
-                    MetricPartialFunctionTypes.MAP_CONDITION_SERIES,
-                    MetricPartialFunctionTypes.WINDOW_CONDITION_FN,
-                    MetricPartialFunctionTypes.AGGREGATE_FN,
-                    MetricFunctionTypes.VALUE,
-                    MetricFunctionTypes.MAP_VALUES,
-                    MetricFunctionTypes.MAP_VALUES,
-                    MetricFunctionTypes.AGGREGATE_VALUE,
-                ]:
-                    logger.warning(
-                        f'Unrecognized metric function type while trying to resolve "{metric_to_resolve.id}".'
-                    )
                 metric_fn_direct_configurations.append(
                     MetricComputationConfiguration(
                         metric_configuration=metric_to_resolve,
diff --git a/great_expectations/expectations/metrics/map_metric_provider.py b/great_expectations/expectations/metrics/map_metric_provider.py
--- a/great_expectations/expectations/metrics/map_metric_provider.py
+++ b/great_expectations/expectations/metrics/map_metric_provider.py
@@ -3306,6 +3306,7 @@ def _register_metric_functions(cls):
                         metric_fn_type=MetricFunctionTypes.VALUE,
                     )
                     if metric_fn_type == MetricPartialFunctionTypes.MAP_CONDITION_FN:
+                        # Documentation in "MetricProvider._register_metric_functions()" explains registration protocol.
                         if domain_type == MetricDomainTypes.COLUMN:
                             register_metric(
                                 metric_name=metric_name
@@ -3443,6 +3444,7 @@ def _register_metric_functions(cls):
                         metric_fn_type=MetricFunctionTypes.VALUE,
                     )
                     if metric_fn_type == MetricPartialFunctionTypes.MAP_CONDITION_FN:
+                        # Documentation in "MetricProvider._register_metric_functions()" explains registration protocol.
                         if domain_type == MetricDomainTypes.COLUMN:
                             register_metric(
                                 metric_name=metric_name
@@ -3576,22 +3578,30 @@ def _get_evaluation_dependencies(
         execution_engine: Optional[ExecutionEngine] = None,
         runtime_configuration: Optional[dict] = None,
     ):
-        metric_name = metric.metric_name
+        dependencies: Dict[str, MetricConfiguration] = {}
+
         base_metric_value_kwargs = {
             k: v for k, v in metric.metric_value_kwargs.items() if k != "result_format"
         }
-        dependencies: Dict[str, MetricConfiguration] = {}
 
-        metric_suffix = ".unexpected_count"
+        metric_name: str = metric.metric_name
+
+        metric_suffix: str = ".unexpected_count"
+
+        # Documentation in "MetricProvider._register_metric_functions()" explains registration/dependency protocol.
         if metric_name.endswith(metric_suffix):
-            try:
-                _ = get_metric_provider(
-                    f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
-                    execution_engine,
-                )
-                has_aggregate_fn = True
-            except gx_exceptions.MetricProviderError:
-                has_aggregate_fn = False
+            has_aggregate_fn: bool = False
+
+            if execution_engine is not None:
+                try:
+                    _ = get_metric_provider(
+                        f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
+                        execution_engine,
+                    )
+                    has_aggregate_fn = True
+                except gx_exceptions.MetricProviderError:
+                    pass
+
             if has_aggregate_fn:
                 dependencies["metric_partial_fn"] = MetricConfiguration(
                     metric_name=f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
@@ -3606,15 +3616,22 @@ def _get_evaluation_dependencies(
                 )
 
         # MapMetric uses "condition" metric to build "unexpected_count.aggregate_fn" and other listed metrics as well.
-        for metric_suffix in [
-            f".unexpected_count.{MetricPartialFunctionTypeSuffixes.AGGREGATE_FUNCTION.value}",
-            ".unexpected_value_counts",
-            ".unexpected_index_query",
-            ".unexpected_index_list",
-            ".filtered_row_count",
-            ".unexpected_values",
-            ".unexpected_rows",
-        ]:
+        unexpected_condition_dependent_metric_name_suffixes: List[str] = list(
+            filter(
+                lambda element: metric_name.endswith(element),
+                [
+                    f".unexpected_count.{MetricPartialFunctionTypeSuffixes.AGGREGATE_FUNCTION.value}",
+                    ".unexpected_value_counts",
+                    ".unexpected_index_query",
+                    ".unexpected_index_list",
+                    ".filtered_row_count",
+                    ".unexpected_values",
+                    ".unexpected_rows",
+                ],
+            )
+        )
+        if len(unexpected_condition_dependent_metric_name_suffixes) == 1:
+            metric_suffix = unexpected_condition_dependent_metric_name_suffixes[0]
             if metric_name.endswith(metric_suffix):
                 dependencies["unexpected_condition"] = MetricConfiguration(
                     metric_name=f"{metric_name[:-len(metric_suffix)]}.{MetricPartialFunctionTypeSuffixes.CONDITION.value}",
diff --git a/great_expectations/expectations/metrics/metric_provider.py b/great_expectations/expectations/metrics/metric_provider.py
--- a/great_expectations/expectations/metrics/metric_provider.py
+++ b/great_expectations/expectations/metrics/metric_provider.py
@@ -139,6 +139,30 @@ def _register_metric_functions(cls) -> None:
                     # This is not a metric (valid metrics possess exectly one metric function).
                     return
 
+                """
+                Basic metric implementations (defined by specifying "metric_name" class variable in "metric_class") use
+                either "@metric_value" decorator (with default "metric_fn_type" set to "MetricFunctionTypes.VALUE"); or
+                "@metric_partial" decorator with specification "partial_fn_type=MetricPartialFunctionTypes.AGGREGATE_FN"
+                (which ultimately sets "metric_fn_type" of inner function to this value); or "@column_aggregate_value"
+                decorator (with default "metric_fn_type" set to "MetricFunctionTypes.VALUE"); or (applicable for column
+                domain metrics only) "column_aggregate_partial" decorator with "partial_fn_type" explicitly set to
+                "MetricPartialFunctionTypes.AGGREGATE_FN".  When "metric_fn_type" of metric implementation function is
+                of "aggregate partial" type ("MetricPartialFunctionTypes.AGGREGATE_FN"), underlying backend (e.g., SQL
+                or Spark) employs "deferred execution" (gather computation needs to build execution plan, then execute
+                all computations combined).  Deferred aggregate function calls are bundled (applies to SQL and Spark).
+                To instruct "ExecutionEngine" accordingly, original metric is registered with its "declared" name, but
+                with "metric_provider" function omitted (set to "None"), and additional "AGGREGATE_FN" metric, with its
+                "metric_provider" set to (decorated) implementation function, defined in metric class, is registered.
+                Then "AGGREGATE_FN" metric can specified with key "metric_partial_fn" as evaluation metric dependency.
+                By convention, aggregate partial metric implementation functions return three-valued tuple, containing
+                deferred execution metric implementation function of corresponding "ExecutionEngine" backend (called
+                "metric_aggregate") as well as "compute_domain_kwargs" and "accessor_domain_kwargs", which are relevant
+                for bundled computation and result access, respectively.  When "ExecutionEngine.resolve_metrics()" finds
+                no "metric_provider" (metric_fn being "None"), it then obtains this three-valued tuple from dictionary
+                of "resolved_metric_dependencies_by_metric_name" using previously declared "metric_partial_fn" key (as
+                described above), composes full metric execution configuration structure, and adds this configuration
+                to list of metrics to be resolved as one bundle (specifics pertaining to "ExecutionEngine" subclasses).
+                """
                 if metric_fn_type not in [
                     MetricFunctionTypes.VALUE,
                     MetricPartialFunctionTypes.AGGREGATE_FN,
@@ -217,22 +241,22 @@ def _get_evaluation_dependencies(
         runtime_configuration: Optional[dict] = None,
     ):
         dependencies: Dict[str, MetricConfiguration] = {}
-        if execution_engine is not None:
-            metric_name = metric.metric_name
-            try:
-                _ = get_metric_provider(
-                    f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
-                    execution_engine,
-                )
-                has_aggregate_fn = True
-            except gx_exceptions.MetricProviderError:
-                has_aggregate_fn = False
-
-            if has_aggregate_fn:
-                dependencies["metric_partial_fn"] = MetricConfiguration(
-                    metric_name=f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
-                    metric_domain_kwargs=metric.metric_domain_kwargs,
-                    metric_value_kwargs=metric.metric_value_kwargs,
-                )
+
+        if execution_engine is None:
+            return dependencies
+
+        try:
+            metric_name: str = metric.metric_name
+            _ = get_metric_provider(
+                f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
+                execution_engine,
+            )
+            dependencies["metric_partial_fn"] = MetricConfiguration(
+                metric_name=f"{metric_name}.{MetricPartialFunctionTypes.AGGREGATE_FN.metric_suffix}",
+                metric_domain_kwargs=metric.metric_domain_kwargs,
+                metric_value_kwargs=metric.metric_value_kwargs,
+            )
+        except gx_exceptions.MetricProviderError:
+            pass
 
         return dependencies


Test patch: