Repository: fairlearn/fairlearn

Issue:
Bug in mitigation using Fairlearn on Azure AutoML models
<!--
Before submitting a bug, please make sure the issue hasn't been already
addressed by searching through the past issues.

If your issue is a usage question, please submit it in one of these other
channels instead:
- StackOverflow with the `fairlearn` tag:
  https://stackoverflow.com/questions/tagged/fairlearn
- Discord: https://discord.gg/R22yCfgsRn
The issue tracker is used only to report bugs and feature requests. For
questions, please use either of the above platforms. Most question issues are
closed without an answer on this issue tracker. Thanks for your understanding.
-->

#### Describe the bug

`GridSearch` is being too restrictive when validating its inputs, rejecting data which it should be able to handle

#### Steps/Code to Reproduce

```python
from sklearn.model_selection import train_test_split
from fairlearn.reductions import GridSearch
from fairlearn.reductions import DemographicParity, ErrorRate
from fairlearn.metrics import MetricFrame, selection_rate, count
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn import metrics as skm
import pandas as pd


from sklearn.datasets import fetch_openml
data = fetch_openml(data_id=1590, as_frame=True)
X_raw = data.data
Y = (data.target == '>50K') * 1


A = X_raw["sex"]
X = X_raw.drop(labels=['sex'], axis=1)

le = LabelEncoder()
Y = le.fit_transform(Y)

X_train, X_test, Y_train, Y_test, A_train, A_test = train_test_split(X,
                                                                     Y,
                                                                     A,
                                                                     test_size=0.2,
                                                                     random_state=0,
                                                                     stratify=Y)

# Work around indexing bug
X_train = X_train.reset_index(drop=True)
A_train = A_train.reset_index(drop=True)
X_test = X_test.reset_index(drop=True)
A_test = A_test.reset_index(drop=True)



# The following shows that LGBMClassifier can handle this dataset
# without the scaling and one-hot encoding used in other examples
from lightgbm import LGBMClassifier

unmitigated_predictor = LGBMClassifier()
unmitigated_predictor.fit(X_train, Y_train)

metric_frame = MetricFrame(metrics={"accuracy": skm.accuracy_score,
                                    "selection_rate": selection_rate,
                                    "count": count},
                           sensitive_features=A_test,
                           y_true=Y_test,
                           y_pred=unmitigated_predictor.predict(X_test))
print(metric_frame.overall)
print(metric_frame.by_group)

# However a GridSearch fails:
sweep = GridSearch(LGBMClassifier(),
                   constraints=DemographicParity(),
                   grid_size=71)

sweep.fit(X_train, Y_train,
          sensitive_features=A_train)

```

#### Expected Results

That the `sweep.fit()` call works

#### Actual Results

```
c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
    709             try:
--> 710                 array = array.astype(np.float64)
    711             except ValueError as e:

ValueError: could not convert string to float: 'Private'

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
<ipython-input-9-cb592bed5a23> in <module>
      1 sweep.fit(X_train, Y_train,
----> 2           sensitive_features=A_train)
      3 
      4 predictors = sweep.predictors_

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\reductions\_grid_search\grid_search.py in fit(self, X, y, **kwargs)
    135         # Prep the parity constraints and objective
    136         logger.debug("Preparing constraints and objective")
--> 137         self.constraints.load_data(X, y, **kwargs)
    138         objective = self.constraints.default_objective()
    139         objective.load_data(X, y, **kwargs)

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\reductions\_moments\utility_parity.py in load_data(self, X, y, sensitive_features, control_features)
    309                                          enforce_binary_labels=True,
    310                                          sensitive_features=sensitive_features,
--> 311                                          control_features=control_features)
    312 
    313         base_event = pd.Series(data=_ALL, index=y_train.index)

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\utils\_input_validation.py in _validate_and_reformat_input(X, y, expect_y, enforce_binary_labels, **kwargs)
     70             y = y.to_numpy().reshape(-1)
     71 
---> 72         X, y = check_X_y(X, y)
     73         y = check_array(y, ensure_2d=False, dtype='numeric')
     74         if enforce_binary_labels and not set(np.unique(y)).issubset(set([0, 1])):
```

Obviously our call to `check_X_y()` is too restrictive.

Comments:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 6d54605..cb8fb19 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,8 @@
 
 ### v0.7.1
 
+* Relaxed checks made on `X` in `_validate_and_reformat_input()` since that
+  is the concern of the underlying estimator and not Fairlearn
 
 
 ### v0.7.0
diff --git a/fairlearn/utils/_input_validation.py b/fairlearn/utils/_input_validation.py
index 962b348..a28cd9d 100644
--- a/fairlearn/utils/_input_validation.py
+++ b/fairlearn/utils/_input_validation.py
@@ -69,7 +69,7 @@ def _validate_and_reformat_input(X, y=None, expect_y=True, enforce_binary_labels
         elif isinstance(y, pd.DataFrame) and y.shape[1] == 1:
             y = y.to_numpy().reshape(-1)
 
-        X, y = check_X_y(X, y)
+        X, y = check_X_y(X, y, dtype=None, force_all_finite=False)
         y = check_array(y, ensure_2d=False, dtype='numeric')
         if enforce_binary_labels and not set(np.unique(y)).issubset(set([0, 1])):
             raise ValueError(_LABELS_NOT_0_1_ERROR_MESSAGE)


Test patch:
diff --git a/test/unit/utils/test_utils.py b/test/unit/utils/test_utils.py
new file mode 100644
index 0000000..f879158
--- /dev/null
+++ b/test/unit/utils/test_utils.py
@@ -0,0 +1,34 @@
+# Copyright (c) Fairlearn contributors.
+# Licensed under the MIT License.
+
+import numpy as np
+import pandas as pd
+
+import fairlearn.utils._input_validation as iv
+
+
+class TestValidateAndReformatInput:
+    def test_smoke(self):
+        # Regression test for Issue #898
+        X = pd.DataFrame.from_dict(
+            {
+                'alpha': ['a', 'a', 'b'],
+                'beta': [1, 2, 1]
+            }
+        )
+        sf = np.asarray(['C', 'D', 'C'])
+        y = np.asarray([0, 0, 1])
+
+        X_update, y_update, sf_update, cf_update = iv._validate_and_reformat_input(
+            X=X,
+            y=y,
+            sensitive_features=sf
+        )
+        assert isinstance(X_update, pd.DataFrame)
+        assert isinstance(y_update, pd.Series)
+        assert isinstance(y_update, pd.Series)
+        assert cf_update is None
+
+        assert np.array_equal(X, X_update)
+        assert np.array_equal(y, y_update)
+        assert np.array_equal(sf, sf_update)