Repository: autogluon/autogluon
Pull request: Validate unique indices for various API
*Issue #, if available:*
https://github.com/awslabs/autogluon/issues/1418

*Description of changes:*
Raise exception if duplicate index found in the data. Previously this check was done only for `fit`, added this check to other APIs

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.


Problem statement:
Error on feature_importance
Hi.
I use the latest version of the library. I trained the model.
When I try to call the feature_importance method like this I get the result and everything is ok (87527 - number of rows).
`my_model.feature_importance(data=testset, subsample_size=87527, num_shuffle_sets=30)`

When I try to reduce the subsample_size like this 
`my_model.feature_importance(data=testset, subsample_size=10000, num_shuffle_sets=30)`
I get this error:
> ValueError: Found input variables with inconsistent numbers of samples: [222191, 10000]

If subsample_size=1000 this error:
> ValueError: Found input variables with inconsistent numbers of samples: [21537, 1000]

What does this error mean? What do these numbers 21537, 222191 mean?

Hints from issue discussion:
Hello! The same with me. I fit a model with 'XGB', 'CAT', 'RF', 'XT', 'FASTAI' as hyperparameters. When try to get feature_importance() I receive the same:

`ValueError: Found input variables with inconsistent numbers of samples: [28087, 1000]`

feature_importance() work only if I set subsample_size=None which means no subsample and I can not wait for the result if data is rather big.

Here is the traceback:

```
Computing feature importance via permutation shuffling for 23 features using 1000 rows with 5 shuffle sets...
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-18-3a3adc27c70f> in <module>()
      2 x_and_y = pd.concat([test_X, test_Y], axis=1)
      3 
----> 4 fi = cls.feature_importance(data=x_and_y, subsample_size=1000, num_shuffle_sets=5, silent=False)
      5 print(fi)

13 frames
/usr/local/lib/python3.7/dist-packages/autogluon/tabular/predictor/predictor.py in feature_importance(self, data, model, features, feature_stage, subsample_size, time_limit, num_shuffle_sets, include_confidence_band, silent)
   1454 
   1455         fi_df = self._learner.get_feature_importance(model=model, X=data, features=features, feature_stage=feature_stage,
-> 1456                                                      subsample_size=subsample_size, time_limit=time_limit, num_shuffle_sets=num_shuffle_sets, silent=silent)
   1457 
   1458         if include_confidence_band:

/usr/local/lib/python3.7/dist-packages/autogluon/tabular/learner/abstract_learner.py in get_feature_importance(self, model, X, y, features, feature_stage, subsample_size, silent, **kwargs)
    681 
    682             if feature_stage == 'original':
--> 683                 return trainer._get_feature_importance_raw(model=model, X=X, y=y, features=features, subsample_size=subsample_size, transform_func=self.transform_features, silent=silent, **kwargs)
    684             X = self.transform_features(X)
    685         else:

/usr/local/lib/python3.7/dist-packages/autogluon/tabular/trainer/abstract_trainer.py in _get_feature_importance_raw(self, X, y, model, eval_metric, **kwargs)
   1506         predict_func_kwargs = dict(model=model)
   1507         return compute_permutation_feature_importance(
-> 1508             X=X, y=y, predict_func=predict_func, predict_func_kwargs=predict_func_kwargs, eval_metric=eval_metric, **kwargs
   1509         )
   1510 

/usr/local/lib/python3.7/dist-packages/autogluon/core/utils/utils.py in compute_permutation_feature_importance(X, y, predict_func, eval_metric, features, subsample_size, num_shuffle_sets, predict_func_kwargs, transform_func, transform_func_kwargs, time_limit, silent, log_prefix, importance_as_list)
    727             X_transformed = X if transform_func is None else transform_func(X, **transform_func_kwargs)
    728             y_pred = predict_func(X_transformed, **predict_func_kwargs)
--> 729             score_baseline = eval_metric(y, y_pred)
    730             if shuffle_repeat == 0:
    731                 if not silent:

/usr/local/lib/python3.7/dist-packages/autogluon/core/metrics/__init__.py in __call__(self, y_true, y_pred, sample_weight)
    126         else:
    127             return self._sign * self._score_func(y_true, y_pred,
--> 128                                                  **self._kwargs)
    129 
    130     @property

/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs)
     61             extra_args = len(args) - len(all_args)
     62             if extra_args <= 0:
---> 63                 return f(*args, **kwargs)
     64 
     65             # extra_args > 0

/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py in f1_score(y_true, y_pred, labels, pos_label, average, sample_weight, zero_division)
   1072                        pos_label=pos_label, average=average,
   1073                        sample_weight=sample_weight,
-> 1074                        zero_division=zero_division)
   1075 
   1076 

/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs)
     61             extra_args = len(args) - len(all_args)
     62             if extra_args <= 0:
---> 63                 return f(*args, **kwargs)
     64 
     65             # extra_args > 0

/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py in fbeta_score(y_true, y_pred, beta, labels, pos_label, average, sample_weight, zero_division)
   1200                                                  warn_for=('f-score',),
   1201                                                  sample_weight=sample_weight,
-> 1202                                                  zero_division=zero_division)
   1203     return f
   1204 

/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs)
     61             extra_args = len(args) - len(all_args)
     62             if extra_args <= 0:
---> 63                 return f(*args, **kwargs)
     64 
     65             # extra_args > 0

/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py in precision_recall_fscore_support(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight, zero_division)
   1463         raise ValueError("beta should be >=0 in the F-beta score")
   1464     labels = _check_set_wise_labels(y_true, y_pred, average, labels,
-> 1465                                     pos_label)
   1466 
   1467     # Calculate tp_sum, pred_sum, true_sum ###

/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py in _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)
   1275                          str(average_options))
   1276 
-> 1277     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
   1278     # Convert to Python primitive type to avoid NumPy type / Python str
   1279     # comparison. See https://github.com/numpy/numpy/issues/6784

/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py in _check_targets(y_true, y_pred)
     81     y_pred : array or indicator matrix
     82     """
---> 83     check_consistent_length(y_true, y_pred)
     84     type_true = type_of_target(y_true)
     85     type_pred = type_of_target(y_pred)

/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in check_consistent_length(*arrays)
    318     if len(uniques) > 1:
    319         raise ValueError("Found input variables with inconsistent numbers of"
--> 320                          " samples: %r" % [int(l) for l in lengths])
    321 
    322 

ValueError: Found input variables with inconsistent numbers of samples: [28087, 1000]
```
Can you try the following as a sanity check:

```
data = data.reset_index(drop=True)
predictor.feature_importance(data, subsample_size=...)
```

I suspect you have duplicate indices in your data, and if so the above code should fix the problem. This is something I plan to fix for v0.4 but want to ensure this is the problem you are facing.
Yep, removing dups solve the problem. I have duplicate indicies in my data, but its ok for my pipeline - index may contain duplicate entries
Also, in almost every run I do not see any printouts from feature_importance() even when I set silent=False. I use autogluon in Google Colab, maybe it is about dependencies or packages installed after autogluon. In feature_importance() its very convenient to see expected runtime but I do not see it. Sounds like 'silent=' param doesnt work as intended. Thank you!

Commit message: remove unnecessary checks

Patch to review:
diff --git a/tabular/src/autogluon/tabular/predictor/predictor.py b/tabular/src/autogluon/tabular/predictor/predictor.py
index d4678bfa37fe..a6e8dc4da325 100644
--- a/tabular/src/autogluon/tabular/predictor/predictor.py
+++ b/tabular/src/autogluon/tabular/predictor/predictor.py
@@ -1228,6 +1228,8 @@ def fit_pseudolabel(self, pseudo_data: pd.DataFrame, max_iter: int = 5, return_p
         if len(pseudo_data) < 1:
             raise Exception('No pseudo data given')
 
+        self._validate_unique_indices(pseudo_data, 'pseudo_data')
+
         if not self._learner.is_fit:
             if 'train_data' not in kwargs.keys():
                 Exception('Autogluon is required to be fit or given \'train_data\' in order to run \'fit_pseudolabel\'.'
@@ -1861,9 +1863,7 @@ def feature_importance(self, data=None, model=None, features=None, feature_stage
         if (data is None) and (not self._trainer.is_data_saved):
             raise AssertionError(
                 'No data was provided and there is no cached data to load for feature importance calculation. `cache_data=True` must be set in the `TabularPredictor` init `learner_kwargs` argument call to enable this functionality when data is not specified.')
-        if data is not None:
-            # Avoid crash when indices are duplicated
-            data = data.reset_index(drop=True)
+        self._validate_unique_indices(data, 'data')
 
         if num_shuffle_sets is None:
             num_shuffle_sets = 10 if time_limit else 3


Code review comments:
diff_hunk: @@ -1861,9 +1863,7 @@ def feature_importance(self, data=None, model=None, features=None, feature_stage
         if (data is None) and (not self._trainer.is_data_saved):
             raise AssertionError(
                 'No data was provided and there is no cached data to load for feature importance calculation. `cache_data=True` must be set in the `TabularPredictor` init `learner_kwargs` argument call to enable this functionality when data is not specified.')
-        if data is not None:
-            # Avoid crash when indices are duplicated
-            data = data.reset_index(drop=True)
+        self._validate_unique_indices(data, 'data')
path: tabular/src/autogluon/tabular/predictor/predictor.py
text: @user1:
data can be None here, only validate if data is not None, otherwise will crash.

```
if data is not None:
    self._validate_unique_indices(data, 'data')
```