Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/expectations/core/expect_table_columns_to_match_set.py b/great_expectations/expectations/core/expect_table_columns_to_match_set.py
--- a/great_expectations/expectations/core/expect_table_columns_to_match_set.py
+++ b/great_expectations/expectations/core/expect_table_columns_to_match_set.py
@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from great_expectations.core import ExpectationConfiguration
 from great_expectations.exceptions import InvalidExpectationConfigurationError
@@ -8,6 +8,18 @@
 from great_expectations.render.renderer.renderer import renderer
 from great_expectations.render.types import RenderedStringTemplateContent
 from great_expectations.render.util import substitute_none_for_missing
+from great_expectations.rule_based_profiler.config import (
+    ParameterBuilderConfig,
+    RuleBasedProfilerConfig,
+)
+from great_expectations.rule_based_profiler.types import (
+    DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME,
+    FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY,
+    FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER,
+    FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY,
+    PARAMETER_KEY,
+    VARIABLES_KEY,
+)
 
 
 class ExpectTableColumnsToMatchSet(TableExpectation):
@@ -60,13 +72,61 @@ class ExpectTableColumnsToMatchSet(TableExpectation):
     success_keys = (
         "column_set",
         "exact_match",
+        "auto",
+        "profiler_config",
     )
+
+    mean_table_columns_set_match_multi_batch_parameter_builder_config: ParameterBuilderConfig = ParameterBuilderConfig(
+        module_name="great_expectations.rule_based_profiler.parameter_builder",
+        class_name="MeanTableColumnsSetMatchMultiBatchParameterBuilder",
+        name="column_names_set_estimator",
+        metric_domain_kwargs=DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME,
+        metric_value_kwargs=None,
+        evaluation_parameter_builder_configs=None,
+        json_serialize=True,
+    )
+    validation_parameter_builder_configs: List[ParameterBuilderConfig] = [
+        mean_table_columns_set_match_multi_batch_parameter_builder_config,
+    ]
+    default_profiler_config: RuleBasedProfilerConfig = RuleBasedProfilerConfig(
+        name="expect_table_columns_to_match_set",  # Convention: use "expectation_type" as profiler name.
+        config_version=1.0,
+        variables={},
+        rules={
+            "expect_table_columns_to_match_set": {
+                "variables": {
+                    "exact_match": None,
+                    "success_ratio": 1.0,
+                },
+                "domain_builder": {
+                    "class_name": "TableDomainBuilder",
+                    "module_name": "great_expectations.rule_based_profiler.domain_builder",
+                },
+                "expectation_configuration_builders": [
+                    {
+                        "expectation_type": "expect_table_columns_to_match_set",
+                        "class_name": "DefaultExpectationConfigurationBuilder",
+                        "module_name": "great_expectations.rule_based_profiler.expectation_configuration_builder",
+                        "validation_parameter_builder_configs": validation_parameter_builder_configs,
+                        "column_set": f"{PARAMETER_KEY}{mean_table_columns_set_match_multi_batch_parameter_builder_config.name}{FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER}{FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY}",
+                        "condition": f"{PARAMETER_KEY}{mean_table_columns_set_match_multi_batch_parameter_builder_config.name}{FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER}{FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY}{FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER}success_ratio >= {VARIABLES_KEY}success_ratio",
+                        "meta": {
+                            "profiler_details": f"{PARAMETER_KEY}{mean_table_columns_set_match_multi_batch_parameter_builder_config.name}{FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER}{FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY}",
+                        },
+                    },
+                ],
+            },
+        },
+    )
+
     default_kwarg_values = {
         "column_set": None,
         "exact_match": True,
         "result_format": "BASIC",
         "include_config": True,
         "catch_exceptions": False,
+        "auto": False,
+        "profiler_config": default_profiler_config,
     }
     args_keys = (
         "column_set",
diff --git a/great_expectations/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py b/great_expectations/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py
--- a/great_expectations/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py
+++ b/great_expectations/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py
@@ -26,6 +26,11 @@ class MeanTableColumnsSetMatchMultiBatchParameterBuilder(
 ):
     """
     Compute mean match ratio (as a fraction) of "table.columns" metric across every Batch of data given.
+
+    Step-1: Compute "table.columns" metric value for each Batch object.
+    Step-2: Compute set union operation of column lists from Step-1 over all Batch objects (gives maximum column set).
+    Step-3: Assign match scores: if column set of a Batch equals overall (maximum) column set, give it 1; 0 otherwise.
+    Step-4: Compute mean value of match scores as "success_ratio" (divide sum of scores by number of Batch objects).
     """
 
     exclude_field_names: Set[
@@ -134,9 +139,9 @@ def _build_parameters(
 
         return Attributes(
             {
-                FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY: mean_table_columns_set_match,
-                FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY: parameter_node[
-                    FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY
-                ],
+                FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY: multi_batch_table_columns_names_as_set,
+                FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY: {
+                    "success_ratio": mean_table_columns_set_match,
+                },
             }
         )


Test patch:
diff --git a/tests/integration/profiling/rule_based_profilers/test_profiler_user_workflows.py b/tests/integration/profiling/rule_based_profilers/test_profiler_user_workflows.py
--- a/tests/integration/profiling/rule_based_profilers/test_profiler_user_workflows.py
+++ b/tests/integration/profiling/rule_based_profilers/test_profiler_user_workflows.py
@@ -1562,6 +1562,62 @@ def test_bobster_expect_table_row_count_to_be_between_auto_yes_default_profiler_
     assert result.expectation_config.kwargs["auto"]
 
 
+@pytest.mark.skipif(
+    version.parse(np.version.version) < version.parse("1.21.0"),
+    reason="requires numpy version 1.21.0 or newer",
+)
+def test_quentin_expect_expect_table_columns_to_match_set_auto_yes_default_profiler_config_yes_custom_profiler_config_no(
+    quentin_validator: Validator,
+):
+    validator: Validator = quentin_validator
+
+    # Use all batches, loaded by Validator, for estimating Expectation argument values.
+    result: ExpectationValidationResult = validator.expect_table_columns_to_match_set(
+        result_format="SUMMARY",
+        include_config=True,
+        auto=True,
+    )
+    assert result.success
+
+    value: Any
+    expectation_config_kwargs: dict = {
+        key: value
+        for key, value in result.expectation_config["kwargs"].items()
+        if key != "column_set"
+    }
+    assert expectation_config_kwargs == {
+        "result_format": "SUMMARY",
+        "include_config": True,
+        "auto": True,
+        "batch_id": "84000630d1b69a0fe870c94fb26a32bc",
+    }
+
+    column_set_expected: List[str] = [
+        "total_amount",
+        "tip_amount",
+        "payment_type",
+        "pickup_datetime",
+        "trip_distance",
+        "dropoff_location_id",
+        "improvement_surcharge",
+        "vendor_id",
+        "tolls_amount",
+        "congestion_surcharge",
+        "rate_code_id",
+        "pickup_location_id",
+        "extra",
+        "fare_amount",
+        "mta_tax",
+        "dropoff_datetime",
+        "store_and_fwd_flag",
+        "passenger_count",
+    ]
+    column_set_computed: List[str] = result.expectation_config["kwargs"]["column_set"]
+
+    assert len(column_set_computed) == len(column_set_expected)
+    assert set(column_set_computed) == set(column_set_expected)
+
+
 @pytest.mark.skipif(
     version.parse(np.version.version) < version.parse("1.21.0"),
     reason="requires numpy version 1.21.0 or newer",
diff --git a/tests/rule_based_profiler/parameter_builder/test_mean_table_columns_set_match_multi_batch_parameter_builder.py b/tests/rule_based_profiler/parameter_builder/test_mean_table_columns_set_match_multi_batch_parameter_builder.py
--- a/tests/rule_based_profiler/parameter_builder/test_mean_table_columns_set_match_multi_batch_parameter_builder.py
+++ b/tests/rule_based_profiler/parameter_builder/test_mean_table_columns_set_match_multi_batch_parameter_builder.py
@@ -11,6 +11,7 @@
 )
 from great_expectations.rule_based_profiler.types import (
     DOMAIN_KWARGS_PARAMETER_FULLY_QUALIFIED_NAME,
+    FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY,
     Domain,
     ParameterContainer,
     ParameterNode,
@@ -70,6 +71,32 @@ def test_execution_mean_table_columns_set_match_multi_batch_parameter_builder(
         domain.id: parameter_container,
     }
 
+    expected_parameter_value: dict = {
+        "value": {
+            "VendorID",
+            "pickup_datetime",
+            "total_amount",
+            "congestion_surcharge",
+            "dropoff_datetime",
+            "mta_tax",
+            "store_and_fwd_flag",
+            "tip_amount",
+            "trip_distance",
+            "payment_type",
+            "DOLocationID",
+            "improvement_surcharge",
+            "extra",
+            "tolls_amount",
+            "RatecodeID",
+            "passenger_count",
+            "PULocationID",
+            "fare_amount",
+        },
+        "details": {
+            "success_ratio": 1.0,
+        },
+    }
+
     mean_table_columns_set_match_multi_batch_parameter_builder.build_parameters(
         domain=domain,
         variables=variables,
@@ -77,19 +104,6 @@ def test_execution_mean_table_columns_set_match_multi_batch_parameter_builder(
         batch_request=batch_request,
     )
 
-    expected_parameter_value: dict = {
-        "value": 1.0,
-        "details": {
-            "metric_configuration": {
-                "metric_name": "table.columns",
-                "domain_kwargs": {},
-                "metric_value_kwargs": None,
-                "metric_dependencies": None,
-            },
-            "num_batches": 3,
-        },
-    }
-
     parameter_node: ParameterNode = get_parameter_value_and_validate_return_type(
         domain=domain,
         parameter_reference=mean_table_columns_set_match_multi_batch_parameter_builder.fully_qualified_parameter_name,
@@ -98,4 +112,11 @@ def test_execution_mean_table_columns_set_match_multi_batch_parameter_builder(
         parameters=parameters,
     )
 
+    assert len(parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY]) == len(
+        expected_parameter_value[FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY]
+    )
+
+    parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY] = set(
+        parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY]
+    )
     assert parameter_node == expected_parameter_value