Repository: fairlearn/fairlearn

Issue:
Bug in mitigation using Fairlearn on Azure AutoML models
<!--
Before submitting a bug, please make sure the issue hasn't been already
addressed by searching through the past issues.

If your issue is a usage question, please submit it in one of these other
channels instead:
- StackOverflow with the `fairlearn` tag:
  https://stackoverflow.com/questions/tagged/fairlearn
- Discord: https://discord.gg/R22yCfgsRn
The issue tracker is used only to report bugs and feature requests. For
questions, please use either of the above platforms. Most question issues are
closed without an answer on this issue tracker. Thanks for your understanding.
-->

#### Describe the bug

`GridSearch` is being too restrictive when validating its inputs, rejecting data which it should be able to handle

#### Steps/Code to Reproduce

```python
from sklearn.model_selection import train_test_split
from fairlearn.reductions import GridSearch
from fairlearn.reductions import DemographicParity, ErrorRate
from fairlearn.metrics import MetricFrame, selection_rate, count
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn import metrics as skm
import pandas as pd


from sklearn.datasets import fetch_openml
data = fetch_openml(data_id=1590, as_frame=True)
X_raw = data.data
Y = (data.target == '>50K') * 1


A = X_raw["sex"]
X = X_raw.drop(labels=['sex'], axis=1)

le = LabelEncoder()
Y = le.fit_transform(Y)

X_train, X_test, Y_train, Y_test, A_train, A_test = train_test_split(X,
                                                                     Y,
                                                                     A,
                                                                     test_size=0.2,
                                                                     random_state=0,
                                                                     stratify=Y)

# Work around indexing bug
X_train = X_train.reset_index(drop=True)
A_train = A_train.reset_index(drop=True)
X_test = X_test.reset_index(drop=True)
A_test = A_test.reset_index(drop=True)



# The following shows that LGBMClassifier can handle this dataset
# without the scaling and one-hot encoding used in other examples
from lightgbm import LGBMClassifier

unmitigated_predictor = LGBMClassifier()
unmitigated_predictor.fit(X_train, Y_train)

metric_frame = MetricFrame(metrics={"accuracy": skm.accuracy_score,
                                    "selection_rate": selection_rate,
                                    "count": count},
                           sensitive_features=A_test,
                           y_true=Y_test,
                           y_pred=unmitigated_predictor.predict(X_test))
print(metric_frame.overall)
print(metric_frame.by_group)

# However a GridSearch fails:
sweep = GridSearch(LGBMClassifier(),
                   constraints=DemographicParity(),
                   grid_size=71)

sweep.fit(X_train, Y_train,
          sensitive_features=A_train)

```

#### Expected Results

That the `sweep.fit()` call works

#### Actual Results

```
c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
    709             try:
--> 710                 array = array.astype(np.float64)
    711             except ValueError as e:

ValueError: could not convert string to float: 'Private'

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
<ipython-input-9-cb592bed5a23> in <module>
      1 sweep.fit(X_train, Y_train,
----> 2           sensitive_features=A_train)
      3 
      4 predictors = sweep.predictors_

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\reductions\_grid_search\grid_search.py in fit(self, X, y, **kwargs)
    135         # Prep the parity constraints and objective
    136         logger.debug("Preparing constraints and objective")
--> 137         self.constraints.load_data(X, y, **kwargs)
    138         objective = self.constraints.default_objective()
    139         objective.load_data(X, y, **kwargs)

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\reductions\_moments\utility_parity.py in load_data(self, X, y, sensitive_features, control_features)
    309                                          enforce_binary_labels=True,
    310                                          sensitive_features=sensitive_features,
--> 311                                          control_features=control_features)
    312 
    313         base_event = pd.Series(data=_ALL, index=y_train.index)

c:\users\riedgar\appdata\local\continuum\miniconda3\envs\nb-automl-fairlearn\lib\site-packages\fairlearn\utils\_input_validation.py in _validate_and_reformat_input(X, y, expect_y, enforce_binary_labels, **kwargs)
     70             y = y.to_numpy().reshape(-1)
     71 
---> 72         X, y = check_X_y(X, y)
     73         y = check_array(y, ensure_2d=False, dtype='numeric')
     74         if enforce_binary_labels and not set(np.unique(y)).issubset(set([0, 1])):
```

Obviously our call to `check_X_y()` is too restrictive.

Discussion:
romanlutz: Do you want to add a test case? Otherwise someone might come along in the future and undo this without realizing what it was for.
riedgar-ms: > Do you want to add a test case? Otherwise someone might come along in the future and undo this without realizing what it was for.

I shall see what I can concoct. It won't be much of a test, since this is about loosening restrictions.
romanlutz: > > Do you want to add a test case? Otherwise someone might come along in the future and undo this without realizing what it was for.
> 
> I shall see what I can concoct. It won't be much of a test, since this is about loosening restrictions.

That's what I mean 🙂 There should be a test for whatever new scenarios you're enabling, so that it would fail if you remove the changes you're adding here.
riedgar-ms: Test added @romanlutz ..... not one of the best, but does at least make sure that we don't enforce X to be all numeric.
adrinjalali: what do we do with this?

> No hosted parallelism has been purchased or granted. To request a free parallelism grant, please fill out the following form https://aka.ms/azpipelines-parallelism-request


romanlutz: > what do we do with this?
> 
> > No hosted parallelism has been purchased or granted. To request a free parallelism grant, please fill out the following form https://aka.ms/azpipelines-parallelism-request

@adrinjalali this is due to https://github.com/fairlearn/fairlearn/issues/757#issuecomment-881224903 and not a problem. It's just duplicating the build that's run in the other Azure org. We can override for the merge if everything else passes.

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 6d54605..cb8fb19 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,8 @@
 
 ### v0.7.1
 
+* Relaxed checks made on `X` in `_validate_and_reformat_input()` since that
+  is the concern of the underlying estimator and not Fairlearn
 
 
 ### v0.7.0
diff --git a/fairlearn/utils/_input_validation.py b/fairlearn/utils/_input_validation.py
index 962b348..a28cd9d 100644
--- a/fairlearn/utils/_input_validation.py
+++ b/fairlearn/utils/_input_validation.py
@@ -69,7 +69,7 @@ def _validate_and_reformat_input(X, y=None, expect_y=True, enforce_binary_labels
         elif isinstance(y, pd.DataFrame) and y.shape[1] == 1:
             y = y.to_numpy().reshape(-1)
 
-        X, y = check_X_y(X, y)
+        X, y = check_X_y(X, y, dtype=None, force_all_finite=False)
         y = check_array(y, ensure_2d=False, dtype='numeric')
         if enforce_binary_labels and not set(np.unique(y)).issubset(set([0, 1])):
             raise ValueError(_LABELS_NOT_0_1_ERROR_MESSAGE)


Test patch:
diff --git a/test/unit/utils/test_utils.py b/test/unit/utils/test_utils.py
new file mode 100644
index 0000000..f879158
--- /dev/null
+++ b/test/unit/utils/test_utils.py
@@ -0,0 +1,34 @@
+# Copyright (c) Fairlearn contributors.
+# Licensed under the MIT License.
+
+import numpy as np
+import pandas as pd
+
+import fairlearn.utils._input_validation as iv
+
+
+class TestValidateAndReformatInput:
+    def test_smoke(self):
+        # Regression test for Issue #898
+        X = pd.DataFrame.from_dict(
+            {
+                'alpha': ['a', 'a', 'b'],
+                'beta': [1, 2, 1]
+            }
+        )
+        sf = np.asarray(['C', 'D', 'C'])
+        y = np.asarray([0, 0, 1])
+
+        X_update, y_update, sf_update, cf_update = iv._validate_and_reformat_input(
+            X=X,
+            y=y,
+            sensitive_features=sf
+        )
+        assert isinstance(X_update, pd.DataFrame)
+        assert isinstance(y_update, pd.Series)
+        assert isinstance(y_update, pd.Series)
+        assert cf_update is None
+
+        assert np.array_equal(X, X_update)
+        assert np.array_equal(y, y_update)
+        assert np.array_equal(sf, sf_update)