Repository: fairlearn/fairlearn

Issue:
TST: CorrelationRemover is failing estimator checks
#### Describe the bug

`CorrelationRemover` is failing in our unit tests to check for estimator API compatibility: 
```
TypeError: fit() got an unexpected keyword argument 'sample_weight'
```

In a recent scikit-learn commit the following change occurred: https://github.com/scikit-learn/scikit-learn/commit/cbfbd863804da1005eda2fe8ee34ebe6380f9b3a# 
The important part there is https://github.com/scikit-learn/scikit-learn/commit/cbfbd863804da1005eda2fe8ee34ebe6380f9b3a#diff-49f814863816a0e8d199ba647472450ecc59a4375171db23cb1ca36063d6dc25R876-R888 that is, it doesn't check whether `sample_weight` is an argument on `fit` anymore. This seems to be the root cause.

This is failing tests so we are blocked from merging PRs until it's fixed.

#### Steps/Code to Reproduce
`python -m pytest test/unit/preprocessing`

#### Versions
```
System:
    python: 3.8.10 (default, Jun  2 2021, 10:49:15)  [GCC 9.4.0]
executable: /usr/bin/python
   machine: Linux-5.11.0-34-generic-x86_64-with-glibc2.29

Python dependencies:
    Cython: None
matplotlib: 3.4.3
     numpy: 1.21.2
    pandas: 1.3.3
       pip: 20.0.2
     scipy: 1.7.1
setuptools: 45.2.0
   sklearn: 1.0
    tempeh: None

```


@adrinjalali do you have a good intuition on what the fix here should be? Since that technique doesn't yet support `sample_weight` (and we have no plans of adding that anytime soon) it seems wrong to add the argument just to satisfy this check. Wdyt?
FYI @koaning who added `CorrelationRemover`

Comments:

Patch:
diff --git a/docs/user_guide/installation_and_version_guide/v0.7.1.rst b/docs/user_guide/installation_and_version_guide/v0.7.1.rst
index 39f145d..bf7a992 100644
--- a/docs/user_guide/installation_and_version_guide/v0.7.1.rst
+++ b/docs/user_guide/installation_and_version_guide/v0.7.1.rst
@@ -15,3 +15,5 @@ v0.7.1
 * Fixed a bug whereby passing a custom :code:`grid` object to a :code:`GridSearch`
   reduction would result in a :code:`KeyError` if the column names were not ordered
   integers.
+* :class:`~fairlearn.preprocessing.CorrelationRemover` now exposes
+  :code:`n_features_in_` and :code:`feature_names_in_`.
\ No newline at end of file
diff --git a/fairlearn/metrics/_extra_metrics.py b/fairlearn/metrics/_extra_metrics.py
index 474116e..e970f28 100644
--- a/fairlearn/metrics/_extra_metrics.py
+++ b/fairlearn/metrics/_extra_metrics.py
@@ -61,7 +61,7 @@ def _get_labels_for_confusion_matrix(labels, pos_label):
     # Ensure unique_labels has two elements
     if len(unique_labels) == 1:
         if unique_labels[0] == pos_label:
-            unique_labels = [None, pos_label]
+            unique_labels = [np.iinfo(np.int64).min, pos_label]
         else:
             unique_labels.append(pos_label)
     elif len(unique_labels) == 2:
diff --git a/fairlearn/preprocessing/_correlation_remover.py b/fairlearn/preprocessing/_correlation_remover.py
index ceebe78..955f6bc 100644
--- a/fairlearn/preprocessing/_correlation_remover.py
+++ b/fairlearn/preprocessing/_correlation_remover.py
@@ -71,13 +71,16 @@ class CorrelationRemover(BaseEstimator, TransformerMixin):
         if isinstance(X, pd.DataFrame):
             self.lookup_ = {c: i for i, c in enumerate(X.columns)}
             return X.values
+        # correctly handle a 1d input
+        if len(X.shape) == 1:
+            return {0: 0}
         self.lookup_ = {i: i for i in range(X.shape[1])}
         return X
 
     def fit(self, X, y=None):
         """Learn the projection required to make the dataset uncorrelated with sensitive columns."""  # noqa: E501
         self._create_lookup(X)
-        X = check_array(X, estimator=self)
+        X = self._validate_data(X)
         X_use, X_sensitive = self._split_X(X)
         self.sensitive_mean_ = X_sensitive.mean()
         X_s_center = X_sensitive - self.sensitive_mean_
@@ -88,7 +91,9 @@ class CorrelationRemover(BaseEstimator, TransformerMixin):
     def transform(self, X):
         """Transform X by applying the correlation remover."""
         X = check_array(X, estimator=self)
-        check_is_fitted(self, ["beta_", "X_shape_", "lookup_", "sensitive_mean_"])
+        check_is_fitted(
+            self, ["beta_", "X_shape_", "lookup_", "sensitive_mean_"]
+        )
         if self.X_shape_[1] != X.shape[1]:
             raise ValueError(
                 f"The trained data has {self.X_shape_[1]} features while this dataset has "
@@ -100,3 +105,17 @@ class CorrelationRemover(BaseEstimator, TransformerMixin):
         X_use = np.atleast_2d(X_use)
         X_filtered = np.atleast_2d(X_filtered)
         return self.alpha * X_filtered + (1 - self.alpha) * X_use
+
+    def _more_tags(self):
+        return {
+            "_xfail_checks": {
+                "check_parameters_default_constructible": (
+                    "sensitive_feature_ids has to be explicitly set to "
+                    "instantiate this estimator"
+                ),
+                "check_transformer_data_not_an_array": (
+                    "this estimator only accepts pandas dataframes or numpy "
+                    "ndarray as input."
+                )
+            }
+        }


Test patch:
diff --git a/test/unit/metrics/test_extra_metrics.py b/test/unit/metrics/test_extra_metrics.py
index f26cbba..4f68dc1 100644
--- a/test/unit/metrics/test_extra_metrics.py
+++ b/test/unit/metrics/test_extra_metrics.py
@@ -36,17 +36,17 @@ class TestGetLabelsForConfusionMatrix:
         r1 = _get_labels_for_confusion_matrix([-1], None)
         assert np.array_equal(r1, [-1, 1])
         r2 = _get_labels_for_confusion_matrix([1], None)
-        assert np.array_equal(r2, [None, 1])
+        assert np.array_equal(r2, [np.iinfo(np.int64).min, 1])
 
     def test_single_value_numeric_pos_label(self):
         r0 = _get_labels_for_confusion_matrix([0], 3)
         assert np.array_equal(r0, [0, 3])
         r1 = _get_labels_for_confusion_matrix([0], 0)
-        assert np.array_equal(r1, [None, 0])
+        assert np.array_equal(r1, [np.iinfo(np.int64).min, 0])
 
     def test_single_value_alpha_pos_label(self):
         r0 = _get_labels_for_confusion_matrix(['a'], 'a')
-        assert np.array_equal(r0, [None, 'a'])
+        assert np.array_equal(r0, [np.iinfo(np.int64).min, 'a'])
         r1 = _get_labels_for_confusion_matrix(['a'], 0)
         assert np.array_equal(r1, ['a', 0])
 
diff --git a/test/unit/preprocessing/linear_dep_remover/test_sklearn_compat.py b/test/unit/preprocessing/linear_dep_remover/test_sklearn_compat.py
index 257a4fe..eebd8b4 100644
--- a/test/unit/preprocessing/linear_dep_remover/test_sklearn_compat.py
+++ b/test/unit/preprocessing/linear_dep_remover/test_sklearn_compat.py
@@ -4,51 +4,44 @@
 import pytest
 import numpy as np
 import pandas as pd
-from sklearn.utils import estimator_checks
+from sklearn.utils.estimator_checks import parametrize_with_checks
 
 from fairlearn.preprocessing import CorrelationRemover
 
 
-@pytest.mark.parametrize(
-    "test_fn",
+@parametrize_with_checks(
     [
-        # transformer checks
-        estimator_checks.check_transformer_general,
-        estimator_checks.check_transformers_unfitted,
-        # general estimator checks
-        estimator_checks.check_fit2d_predict1d,
-        estimator_checks.check_methods_subset_invariance,
-        estimator_checks.check_fit2d_1sample,
-        estimator_checks.check_fit2d_1feature,
-        estimator_checks.check_get_params_invariance,
-        estimator_checks.check_set_params,
-        estimator_checks.check_dict_unchanged,
-        estimator_checks.check_dont_overwrite_parameters,
-        # nonmeta_checks
-        estimator_checks.check_estimators_dtypes,
-        estimator_checks.check_fit_score_takes_y,
-        estimator_checks.check_dtype_object,
-        estimator_checks.check_sample_weights_pandas_series,
-        estimator_checks.check_sample_weights_list,
-        estimator_checks.check_estimators_fit_returns_self,
-        estimator_checks.check_complex_data,
-        estimator_checks.check_estimators_empty_data_messages,
-        estimator_checks.check_pipeline_consistency,
-        estimator_checks.check_estimators_nan_inf,
-        estimator_checks.check_estimators_overwrite_params,
-        estimator_checks.check_estimator_sparse_data,
-        estimator_checks.check_estimators_pickle,
+        CorrelationRemover(sensitive_feature_ids=[]),
+        CorrelationRemover(sensitive_feature_ids=[0]),
     ]
 )
-def test_estimator_checks(test_fn):
-    test_fn(CorrelationRemover.__name__, CorrelationRemover(sensitive_feature_ids=[]))
-    test_fn(CorrelationRemover.__name__, CorrelationRemover(sensitive_feature_ids=[0]))
+def test_sklearn_compatible_estimator(estimator, check):
+    check(estimator)
 
 
 def test_linear_dependence():
-    X = np.array([[0, 0, 1, 1, ],
-                  [1, 1, 2, 2, ],
-                  [0.1, 0.2, 1.2, 1.1, ]]).T
+    X = np.array(
+        [
+            [
+                0,
+                0,
+                1,
+                1,
+            ],
+            [
+                1,
+                1,
+                2,
+                2,
+            ],
+            [
+                0.1,
+                0.2,
+                1.2,
+                1.1,
+            ],
+        ]
+    ).T
 
     X_tfm = CorrelationRemover(sensitive_feature_ids=[0]).fit(X).transform(X)
     assert X_tfm.shape[1] == 2
@@ -56,12 +49,33 @@ def test_linear_dependence():
 
 
 def test_linear_dependence_pd():
-    X = np.array([[0, 0, 1, 1, ],
-                  [1, 1, 2, 2, ],
-                  [0.1, 0.2, 1.2, 1.1, ]]).T
+    X = np.array(
+        [
+            [
+                0,
+                0,
+                1,
+                1,
+            ],
+            [
+                1,
+                1,
+                2,
+                2,
+            ],
+            [
+                0.1,
+                0.2,
+                1.2,
+                1.1,
+            ],
+        ]
+    ).T
 
-    df = pd.DataFrame(X, columns=['a', 'b', 'c'])
+    df = pd.DataFrame(X, columns=["a", "b", "c"])
 
-    X_tfm = CorrelationRemover(sensitive_feature_ids=['a']).fit(df).transform(df)
+    X_tfm = (
+        CorrelationRemover(sensitive_feature_ids=["a"]).fit(df).transform(df)
+    )
     assert X_tfm.shape[1] == 2
     assert np.allclose(X_tfm[:, 0], 1.5)