Repository: great-expectations/great_expectations

Issue:
New `expect_column_values_to_be_independent` expectation
**Is your feature request related to a problem? Please describe.**
I am working with an insurance dataset. Observations are ordered by year and I would like to check whether our data is stable over time, i.e. the distribution of column values is somewhat constant.

**Describe the solution you'd like**
For this, a `expect_column_values_to_be_independent` expectation would be optimal. For two categorical features (e.g. with `year` as a factor and `type_of_car` in our application) this could be done using a simple [chi squared test](https://en.wikipedia.org/wiki/Chi-squared_test) (see the example) and might reveal that people have been driving more SUVs in recent years.

**Describe alternatives you've considered**
This issue can somehow, but not satisfactory, be tackled using conditional expectations (#1217).



Discussion:
Do you think this is relevant / sensible for GE? Is there already developement of anything similar? I'd be open to have a go at implementing (with some guidance) if there is a general interest.
I think this makes a lot of sense, and would be happy to help working with you to get to a PR!

Resolving patch:
diff --git a/great_expectations/dataset/dataset.py b/great_expectations/dataset/dataset.py
--- a/great_expectations/dataset/dataset.py
+++ b/great_expectations/dataset/dataset.py
@@ -84,27 +84,54 @@ def column_aggregate_expectation(cls, func):
         @wraps(func)
         def inner_wrapper(
             self,
-            column,
+            column=None,
             result_format=None,
             row_condition=None,
             condition_parser=None,
             *args,
             **kwargs
         ):
-
             if result_format is None:
                 result_format = self.default_expectation_args["result_format"]
             # Retain support for string-only output formats:
             result_format = parse_result_format(result_format)
 
-            if row_condition:
-                self = self.query(row_condition).reset_index(drop=True)
+            if row_condition and self._supports_row_condition:
+                self = self.query(row_condition, parser=condition_parser).reset_index(
+                    drop=True
+                )
 
             element_count = self.get_row_count()
-            nonnull_count = self.get_column_nonnull_count(column)
-            null_count = element_count - nonnull_count
 
-            evaluation_result = func(self, column, *args, **kwargs)
+            if kwargs.get("column"):
+                column = kwargs.get("column")
+
+            if column is not None:
+                nonnull_count = self.get_column_nonnull_count(
+                    kwargs.get("column", column)
+                )
+                # column is treated specially as a positional argument in most expectations
+                args = tuple((column, *args))
+            elif kwargs.get("column_A") and kwargs.get("column_B"):
+                try:
+                    nonnull_count = (
+                        self[kwargs.get("column_A")].notnull()
+                        & self[kwargs.get("column_B")].notnull()
+                    ).sum()
+                except TypeError:
+                    nonnull_count = None
+            else:
+                raise ValueError(
+                    "The column_aggregate_expectation wrapper requires either column or "
+                    "both column_A and column_B as input."
+                )
+
+            if nonnull_count:
+                null_count = element_count - nonnull_count
+            else:
+                null_count = None
+
+            evaluation_result = func(self, *args, **kwargs)
 
             if "success" not in evaluation_result:
                 raise ValueError(
@@ -126,12 +153,20 @@ def inner_wrapper(
             return_obj["result"] = {
                 "observed_value": evaluation_result["result"]["observed_value"],
                 "element_count": element_count,
-                "missing_count": null_count,
-                "missing_percent": null_count * 100.0 / element_count
-                if element_count > 0
-                else None,
             }
 
+            if null_count:
+                return_obj["result"]["missing_count"] = null_count
+                if element_count > 0:
+                    return_obj["result"]["missing_percent"] = (
+                        null_count * 100.0 / element_count
+                    )
+                else:
+                    return_obj["result"]["missing_percent"] = None
+            else:
+                return_obj["result"]["missing_count"] = None
+                return_obj["result"]["missing_percent"] = None
+
             if result_format["result_format"] == "BASIC":
                 return return_obj
 
@@ -154,6 +189,7 @@ class Dataset(MetaDataset):
     # This should in general only be changed when a subclass *adds expectations* or *changes expectation semantics*
     # That way, multiple backends can implement the same data_asset_type
     _data_asset_type = "Dataset"
+    _supports_row_condition = False
 
     # getter functions with hashable arguments - can be cached
     hashable_getters = [
@@ -344,6 +380,18 @@ def get_column_count_in_range(
         """Returns: int"""
         raise NotImplementedError
 
+    def get_crosstab(
+        self,
+        column_A,
+        column_B,
+        bins_A=None,
+        bins_B=None,
+        n_bins_A=None,
+        n_bins_B=None,
+    ):
+        """Get crosstab of column_A and column_B, binning values if necessary"""
+        raise NotImplementedError
+
     def test_column_map_expectation_function(self, function, *args, **kwargs):
         """Test a column map expectation function
 
@@ -4198,6 +4246,80 @@ def expect_column_kl_divergence_to_be_less_than(
 
         return return_obj
 
+    @MetaDataset.column_aggregate_expectation
+    def expect_column_pair_cramers_phi_value_to_be_less_than(
+        self,
+        column_A,
+        column_B,
+        bins_A=None,
+        bins_B=None,
+        n_bins_A=None,
+        n_bins_B=None,
+        threshold=0.05,
+        result_format=None,
+        include_config=True,
+        catch_exceptions=None,
+        meta=None,
+    ):
+        """
+        Expect the values in column_A to be independent of those in column_B.
+
+        Args:
+            column_A (str): The first column name
+            column_B (str): The second column name
+            threshold (float): Maximum allowed value of cramers V for expectation to pass.
+
+        Keyword Args:
+            bins_A (list of float): Bins for column_A.
+            bins_B (list of float): Bins for column_B.
+            n_bins_A (int): Number of bins for column_A. Ignored if bins_A is not None.
+            n_bins_B (int): Number of bins for column_B. Ignored if bins_B is not None.
+
+        Other Parameters:
+            result_format (str or None): \
+                Which output mode to use: `BOOLEAN_ONLY`, `BASIC`, `COMPLETE`, or `SUMMARY`.
+                For more detail, see :ref:`result_format <result_format>`.
+            include_config (boolean): \
+                If True, then include the expectation config as part of the result object. \
+                For more detail, see :ref:`include_config`.
+            catch_exceptions (boolean or None): \
+                If True, then catch exceptions and include them as part of the result object. \
+                For more detail, see :ref:`catch_exceptions`.
+            meta (dict or None): \
+                A JSON-serializable dictionary (nesting allowed) that will be included in the output without \
+                modification. For more detail, see :ref:`meta`.
+
+        Returns:
+            A JSON-serializable expectation result object.
+
+            Exact fields vary depending on the values passed to :ref:`result_format <result_format>` and
+            :ref:`include_config`, :ref:`catch_exceptions`, and :ref:`meta`.
+
+        """
+        crosstab = self.get_crosstab(
+            column_A, column_B, bins_A, bins_B, n_bins_A, n_bins_B
+        )
+        chi2_result = stats.chi2_contingency(crosstab)
+        # See e.g. https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V
+        cramers_V = max(
+            min(
+                np.sqrt(
+                    chi2_result[0] / self.get_row_count() / (min(crosstab.shape) - 1)
+                ),
+                1,
+            ),
+            0,
+        )
+        return_obj = {
+            "success": cramers_V <= threshold,
+            "result": {
+                "observed_value": cramers_V,
+                "unexpected_list": crosstab,
+                "details": {"crosstab": crosstab},
+            },
+        }
+        return return_obj
+
     ###
     #
     # Column pairs
@@ -4345,7 +4467,7 @@ def expect_column_pair_values_to_be_in_set(
 
     ###
     #
-    # Multicolumn pairs
+    # Multicolumn
     #
     ###
 
@@ -4390,6 +4512,27 @@ def expect_multicolumn_values_to_be_unique(
         """
         raise NotImplementedError
 
+    def expect_multicolumn_sum_to_equal(
+        self,
+        column_list,
+        sum_total,
+        result_format=None,
+        include_config=True,
+        catch_exceptions=None,
+        meta=None,
+    ):
+        """ Multi-Column Map Expectation
+
+        Expects that sum of all rows for a set of columns is equal to a specific value
+
+        Args:
+            column_list (List[str]): \
+                Set of columns to be checked
+            sum_total (int): \
+                expected sum of columns
+        """
+        raise NotImplementedError
+
     @staticmethod
     def _parse_value_set(value_set):
         parsed_value_set = [
diff --git a/great_expectations/dataset/pandas_dataset.py b/great_expectations/dataset/pandas_dataset.py
--- a/great_expectations/dataset/pandas_dataset.py
+++ b/great_expectations/dataset/pandas_dataset.py
@@ -69,16 +69,10 @@ def inner_wrapper(
                 result_format = self.default_expectation_args["result_format"]
 
             result_format = parse_result_format(result_format)
-            if row_condition:
-                if condition_parser not in ["python", "pandas"]:
-                    raise ValueError(
-                        "condition_parser is required when setting a row_condition,"
-                        " and must be 'python' or 'pandas'"
-                    )
-                else:
-                    data = self.query(
-                        row_condition, parser=condition_parser
-                    ).reset_index(drop=True)
+            if row_condition and self._supports_row_condition:
+                data = self._apply_row_condition(
+                    row_condition=row_condition, condition_parser=condition_parser
+                )
             else:
                 data = self
 
@@ -393,6 +387,7 @@ class PandasDataset(MetaPandasDataset, pd.DataFrame):
         "discard_subset_failing_expectations",
     ]
     _internal_names_set = set(_internal_names)
+    _supports_row_condition = True
 
     # We may want to expand or alter support for subclassing dataframes in the future:
     # See http://pandas.pydata.org/pandas-docs/stable/extending.html#extending-subclassing-pandas
@@ -422,6 +417,17 @@ def __init__(self, *args, **kwargs):
             "discard_subset_failing_expectations", False
         )
 
+    def _apply_row_condition(self, row_condition, condition_parser):
+        if condition_parser not in ["python", "pandas"]:
+            raise ValueError(
+                "condition_parser is required when setting a row_condition,"
+                " and must be 'python' or 'pandas'"
+            )
+        else:
+            return self.query(row_condition, parser=condition_parser).reset_index(
+                drop=True
+            )
+
     def get_row_count(self):
         return self.shape[0]
 
@@ -521,6 +527,57 @@ def get_column_count_in_range(
                 result = result[result <= max_val]
         return len(result)
 
+    def get_crosstab(
+        self,
+        column_A,
+        column_B,
+        bins_A=None,
+        bins_B=None,
+        n_bins_A=None,
+        n_bins_B=None,
+    ):
+        """Get crosstab of column_A and column_B, binning values if necessary"""
+        series_A = self.get_binned_values(self[column_A], bins_A, n_bins_A)
+        series_B = self.get_binned_values(self[column_B], bins_B, n_bins_B)
+        return pd.crosstab(series_A, columns=series_B)
+
+    def get_binned_values(self, series, bins, n_bins):
+        """
+        Get binned values of series.
+
+        Args:
+            Series (pd.Series): Input series
+            bins (list):
+                Bins for the series. List of numeric if series is numeric or list of list
+                of series values else.
+            n_bins (int): Number of bins. Ignored if bins is not None.
+        """
+        if n_bins is None:
+            n_bins = 10
+
+        if series.dtype in ["int", "float"]:
+            if bins is None:
+                bins = np.histogram_bin_edges(series[series.notnull()], bins=n_bins)
+                # Make sure max of series is included in rightmost bin
+                bins[-1] = np.nextafter(bins[-1], bins[-1] + 1)
+
+            # Missings get digitized into bin = n_bins+1
+            return np.digitize(series, bins=bins)
+
+        else:
+            if bins is None:
+                value_counts = series.value_counts(sort=True)
+                if len(value_counts) < n_bins + 1:
+                    return series.fillna("(missing)")
+                else:
+                    other_values = sorted(value_counts.index[n_bins:])
+                    replace = {value: "(other)" for value in other_values}
+            else:
+                replace = dict()
+                for x in bins:
+                    replace.update({value: ", ".join(x) for value in x})
+            return series.replace(to_replace=replace).fillna("(missing)")
+
     ### Expectation methods ###
 
     @DocInherit
diff --git a/great_expectations/render/renderer/content_block/expectation_string.py b/great_expectations/render/renderer/content_block/expectation_string.py
--- a/great_expectations/render/renderer/content_block/expectation_string.py
+++ b/great_expectations/render/renderer/content_block/expectation_string.py
@@ -433,6 +433,32 @@ def expect_table_columns_to_match_ordered_list(
             )
         ]
 
+    @classmethod
+    def expect_column_pair_cramers_phi_value_to_be_less_than(
+        cls, expectation, styling=None, include_column_name=True
+    ):
+        params = substitute_none_for_missing(
+            expectation.kwargs, ["column_A", "column_B"]
+        )
+
+        if (params["column_A"] is None) or (params["column_B"] is None):
+            template_str = " unrecognized kwargs for expect_column_pair_cramers_phi_value_to_be_less_than: missing column."
+
+        template_str = "Values in $column_A and $column_B must be independent."
+
+        return [
+            RenderedStringTemplateContent(
+                **{
+                    "content_block_type": "string_template",
+                    "string_template": {
+                        "template": template_str,
+                        "params": params,
+                        "styling": styling,
+                    },
+                }
+            )
+        ]
+
     @classmethod
     def expect_multicolumn_values_to_be_unique(
         cls, expectation, styling=None, include_column_name=True


Test patch:
diff --git a/tests/data_asset/test_data_asset.py b/tests/data_asset/test_data_asset.py
--- a/tests/data_asset/test_data_asset.py
+++ b/tests/data_asset/test_data_asset.py
@@ -104,6 +104,7 @@ def test_valid_expectation_types(dataset, pandas_dataset):
         "expect_column_median_to_be_between",
         "expect_column_min_to_be_between",
         "expect_column_most_common_value_to_be_in_set",
+        "expect_column_pair_cramers_phi_value_to_be_less_than",
         "expect_column_pair_values_A_to_be_greater_than_B",
         "expect_column_pair_values_to_be_equal",
         "expect_column_pair_values_to_be_in_set",
diff --git a/tests/dataset/test_pandas_dataset_conditionals.py b/tests/dataset/test_pandas_dataset_conditionals.py
--- a/tests/dataset/test_pandas_dataset_conditionals.py
+++ b/tests/dataset/test_pandas_dataset_conditionals.py
@@ -103,7 +103,12 @@ def test_expectation_decorator_summary_mode():
 
     assert (
         df.expect_column_mean_to_be_between(
-            "x", 3, 7, result_format="SUMMARY", row_condition="group=='a'"
+            "x",
+            3,
+            7,
+            result_format="SUMMARY",
+            row_condition="group=='a'",
+            condition_parser="pandas",
         )
         == exp_output
     )
diff --git a/tests/render/test_render.py b/tests/render/test_render.py
--- a/tests/render/test_render.py
+++ b/tests/render/test_render.py
@@ -248,6 +248,7 @@ def test_content_block_list_available_expectations():
         "expect_table_columns_to_match_ordered_list",
         "expect_table_row_count_to_be_between",
         "expect_table_row_count_to_equal",
+        "expect_column_pair_cramers_phi_value_to_be_less_than",
     }
     assert known_validation_results_implemented_expectations <= set(
         available_expectations
diff --git a/tests/test_definitions/column_aggregate_expectations/expect_column_pair_cramers_phi_value_to_be_less_than.json b/tests/test_definitions/column_aggregate_expectations/expect_column_pair_cramers_phi_value_to_be_less_than.json
new file mode 100644
--- /dev/null
+++ b/tests/test_definitions/column_aggregate_expectations/expect_column_pair_cramers_phi_value_to_be_less_than.json
@@ -0,0 +1,222 @@
+{
+  "expectation_type": "expect_column_pair_cramers_phi_value_to_be_less_than",
+  "datasets": [
+    {
+      "data": {
+        "integer_1": [
+          1,
+          2,
+          3,
+          4,
+          1,
+          2,
+          3,
+          4,
+          1,
+          2,
+          3,
+          4
+        ],
+        "integer_2": [
+          1,
+          1,
+          1,
+          1,
+          2,
+          2,
+          2,
+          2,
+          3,
+          3,
+          3,
+          3
+        ],
+        "string_1": [
+          "a",
+          "a",
+          "b",
+          "b",
+          "c",
+          "c",
+          "d",
+          "d",
+          "e",
+          "e",
+          "f",
+          "f"
+        ],
+        "string_2": [
+          "a",
+          "a",
+          "b",
+          "c",
+          "d",
+          "e",
+          "f",
+          "g",
+          "h",
+          "i",
+          "j",
+          "k"
+        ],
+        "string_3": [
+          "a",
+          "a",
+          "a",
+          "a",
+          "c",
+          "c",
+          "c",
+          "c",
+          "d",
+          "d",
+          "d",
+          "d"
+        ],
+        "float": [
+          1.0,
+          1.1,
+          1.2,
+          1.3,
+          2.0,
+          2.1,
+          2.2,
+          2.3,
+          3.0,
+          3.1,
+          3.2,
+          3.3
+        ],
+        "float_with_missings_1": [null, null, null, null, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0],
+        "float_with_missings_2": [null, null, null, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0],
+        "string_with_missings": [null, null, null, "a", "b", "c", "a", "b", "c", "a", "b", "c"]
+      },
+      "tests": [
+        {
+          "title": "Basic failure test",
+          "in": {
+            "column_A": "integer_1",
+            "column_B": "integer_1"
+          },
+          "out": {
+            "success": false,
+            "observed_value": 1
+          },
+          "tolerance": 0.000000000001,
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "Basic pass test",
+          "in": {
+            "column_A": "integer_1",
+            "column_B": "integer_2"
+          },
+          "out": {
+            "success": true,
+            "observed_value": 0
+          },
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "Basic pass int string test",
+          "in": {
+            "column_A": "integer_1",
+            "column_B": "string_3"
+          },
+          "out": {
+            "success": true,
+            "observed_value": 0
+          },
+          "exact_match_out": false,
+           "only_for": ["pandas"]
+        },
+        {
+          "title": "test string n_bins",
+          "in": {
+            "column_A": "string_1",
+            "column_B": "string_2",
+            "n_bins_B": 2
+          },
+          "out": {
+            "success": false,
+            "observed_value": 0.8498365855987975
+          },
+          "exact_match_out": false,
+          "tolerance": 0.000000000001,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "test string preconfigured bins",
+          "in": {
+            "column_A": "string_1",
+            "column_B": "string_2",
+            "bins_B": [
+              [
+                "b",
+                "c"
+              ],
+              [
+                "d",
+                "e"
+              ],
+              [
+                "f",
+                "g"
+              ]
+            ]
+          },
+          "out": {
+            "success": false,
+            "observed_value": 1.0
+          },
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "test float bins preconfigured bins",
+          "in": {
+            "column_A": "integer_1",
+            "column_B": "float",
+            "n_bins_B": 3
+          },
+          "out": {
+            "success": true,
+            "observed_value": 0
+          },
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "test float with missing",
+          "in": {
+            "column_A": "integer_1",
+            "column_B": "float_with_missings_1",
+            "n_bins_B": 2
+          },
+          "out": {
+            "success": true,
+            "observed_value": 0
+          },
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        },
+        {
+          "title": "test float and string with missings",
+          "in": {
+            "column_A": "float_with_missings_2",
+            "column_B": "string_with_missings",
+            "n_bins_A": 3
+          },
+          "out": {
+            "success": false,
+            "observed_value": 0.5773502691896257
+          },
+          "exact_match_out": false,
+          "only_for": ["pandas"]
+        }
+      ]
+    }
+  ]
+}
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -643,6 +643,7 @@ def candidate_test_is_on_temporary_notimplemented_list(context, expectation_type
             "expect_column_pair_values_A_to_be_greater_than_B",
             "expect_column_pair_values_to_be_in_set",
             "expect_multicolumn_values_to_be_unique",
+            "expect_column_pair_cramers_phi_value_to_be_less_than",
             # "expect_table_row_count_to_equal_other_table",
             "expect_multicolumn_sum_to_equal",
         ]
@@ -693,6 +694,7 @@ def candidate_test_is_on_temporary_notimplemented_list(context, expectation_type
             # "expect_column_pair_values_A_to_be_greater_than_B",
             # "expect_column_pair_values_to_be_in_set",
             # "expect_multicolumn_values_to_be_unique",
+            "expect_column_pair_cramers_phi_value_to_be_less_than",
             "expect_table_row_count_to_equal_other_table",
             # "expect_multicolumn_sum_to_equal",
         ]