# Buggy code
def general_data_processing(
    self, X: DataFrame, X_val: DataFrame, holdout_frac: float, num_bagging_folds: int
):
    """General data processing steps used for all models."""
    X = copy.deepcopy(X)
    # TODO: We should probably uncomment the below lines, NaN label should be treated as just another value in multiclass classification -> We will have to remove missing, compute problem type, and add back missing if multiclass
    # if self.problem_type == MULTICLASS:
    #     X[self.label] = X[self.label].fillna('')

    # Remove all examples with missing labels from this dataset:
    missinglabel_inds = [index for index, x in X[self.label].isna().iteritems() if x]
    if len(missinglabel_inds) > 0:
        logger.warning(
            f"Warning: Ignoring {len(missinglabel_inds)} (out of {len(X)}) training examples for which the label value in column '{self.label}' is missing"
        )
        X = X.drop(missinglabel_inds, axis=0)

    if self.problem_type is None:
        self.problem_type = self.infer_problem_type(X[self.label])

    if X_val is not None and self.label in X_val.columns:
        # TODO: This is not an ideal solution, instead check if bagging and X_val exists with label, then merge them prior to entering general data processing.
        #  This solution should handle virtually all cases correctly, only downside is it might cut more classes than it needs to.
        self.threshold, holdout_frac, num_bagging_folds = (
            self.adjust_threshold_if_necessary(
                X[self.label],
                threshold=self.threshold,
                holdout_frac=1,
                num_bagging_folds=num_bagging_folds,
            )
        )
    else:
        self.threshold, holdout_frac, num_bagging_folds = (
            self.adjust_threshold_if_necessary(
                X[self.label],
                threshold=self.threshold,
                holdout_frac=holdout_frac,
                num_bagging_folds=num_bagging_folds,
            )
        )

    if (
        (self.eval_metric is not None)
        and (self.eval_metric.name in ["log_loss", "pac_score"])
        and (self.problem_type == MULTICLASS)
    ):
        X = augment_rare_classes(X, self.label, self.threshold)

    # Gets labels prior to removal of infrequent classes
    y_uncleaned = X[self.label].copy()

    self.cleaner = Cleaner.construct(
        problem_type=self.problem_type, label=self.label, threshold=self.threshold
    )
    # TODO: What if all classes in X are low frequency in multiclass? Currently we would crash. Not certain how many problems actually have this property
    X = self.cleaner.fit_transform(
        X
    )  # TODO: Consider merging cleaner into label_cleaner
    X, y = self.extract_label(X)
    self.label_cleaner = LabelCleaner.construct(
        problem_type=self.problem_type, y=y, y_uncleaned=y_uncleaned
    )
    y = self.label_cleaner.transform(y)

    if self.label_cleaner.num_classes is not None:
        logger.log(20, f"Train Data Class Count: {self.label_cleaner.num_classes}")

    if X_val is not None and self.label in X_val.columns:
        X_val = self.cleaner.transform(X_val)
        if len(X_val) == 0:
            logger.warning(
                "All X_val data contained low frequency classes, ignoring X_val and generating from subset of X"
            )
            X_val = None
            y_val = None
        else:
            X_val, y_val = self.extract_label(X_val)
            y_val = self.label_cleaner.transform(y_val)
    else:
        y_val = None

    # TODO: Move this up to top of data before removing data, this way our feature generator is better
    if X_val is not None:
        # Do this if working with SKLearn models, otherwise categorical features may perform very badly on the test set
        logger.log(
            15,
            "Performing general data preprocessing with merged train & validation data, so validation performance may not accurately reflect performance on new test data",
        )
        X_super = pd.concat([X, X_val], ignore_index=True)
        X_super = self.feature_generator.fit_transform(
            X_super, banned_features=self.submission_columns, drop_duplicates=False
        )
        X = X_super.head(len(X)).set_index(X.index)
        X_val = X_super.tail(len(X_val)).set_index(X_val.index)
        del X_super
    else:
        X = self.feature_generator.fit_transform(
            X, banned_features=self.submission_columns, drop_duplicates=False
        )

    return X, y, X_val, y_val, holdout_frac, num_bagging_folds


# Traceback (ValueError)
File delimiter for ../input/house-prices-advanced-regression-techniques/test.csv inferred as ',' (comma). If this is incorrect, please manually load the data as a pandas DataFrame.
Loaded data from: ../input/house-prices-advanced-regression-techniques/test.csv | Columns = 80 / 80 | Rows = 1459 -> 1459
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-48-91455d7b9940> in <module>
1 test_data = task.Dataset(file_path=os.path.join(in_dir, job, 'test.csv'))
2
----> 3 preds = model.predict(test_data)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/task/tabular_prediction/predictor.py in predict(self, dataset, model, as_pandas, use_pred_cache, add_to_pred_cache)
130         """
131         dataset = self.__get_dataset(dataset)
--> 132         return self._learner.predict(X=dataset, model=model, as_pandas=as_pandas, use_pred_cache=use_pred_cache, add_to_pred_cache=add_to_pred_cache)
133
134     def predict_proba(self, dataset, model=None, as_pandas=False, as_multiclass=False):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in predict(self, X, model, as_pandas, use_pred_cache, add_to_pred_cache)
127
128         if len(X_cache_miss) > 0:
--> 129             y_pred_proba = self.predict_proba(X=X_cache_miss, model=model, inverse_transform=False)
130             problem_type = self.trainer_problem_type or self.problem_type
131             y_pred = get_pred_from_proba(y_pred_proba=y_pred_proba, problem_type=problem_type)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in predict_proba(self, X, model, as_pandas, as_multiclass, inverse_transform)
92     # TODO: Add pred_proba_cache functionality as in predict()
93     def predict_proba(self, X: DataFrame, model=None, as_pandas=False, as_multiclass=False, inverse_transform=True):
---> 94         X = self.transform_features(X)
95         y_pred_proba = self.load_trainer().predict_proba(X, model=model)
96         if inverse_transform:

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in transform_features(self, X)
216     def transform_features(self, X):
217         for feature_generator in self.feature_generators:
--> 218             X = feature_generator.transform(X)
219         return X
220

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/utils/decorators.py in inner1(*args, **kwargs)
12         begin = time.time()
13
---> 14         output = func(*args, **kwargs)
15
16         # storing time after function execution

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/features/abstract_feature_generator.py in transform(self, X)
183             raise ValueError(f'Required columns are missing from the provided dataset. Missing columns: {missing_cols}')
184
--> 185         X = X.astype(self.features_init_types)
186         X.reset_index(drop=True, inplace=True)
187         X_features = self.generate_features(X)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/generic.py in astype(self, dtype, copy, errors, **kwargs)
5863                     results.append(
5864                         col.astype(
-> 5865                             dtype=dtype[col_name], copy=copy, errors=errors, **kwargs
5866                         )
5867                     )

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/generic.py in astype(self, dtype, copy, errors, **kwargs)
5880             # else, only a single dtype is given
5881             new_data = self._data.astype(
-> 5882                 dtype=dtype, copy=copy, errors=errors, **kwargs
5883             )
5884             return self._constructor(new_data).__finalize__(self)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/managers.py in astype(self, dtype, **kwargs)
579
580     def astype(self, dtype, **kwargs):
--> 581         return self.apply("astype", dtype=dtype, **kwargs)
582
583     def convert(self, **kwargs):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/managers.py in apply(self, f, axes, filter, do_integrity_check, consolidate, **kwargs)
436                     kwargs[k] = obj.reindex(b_items, axis=axis, copy=align_copy)
437
--> 438             applied = getattr(b, f)(**kwargs)
439             result_blocks = _extend_blocks(applied, result_blocks)
440

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/blocks.py in astype(self, dtype, copy, errors, values, **kwargs)
557
558     def astype(self, dtype, copy=False, errors="raise", values=None, **kwargs):
--> 559         return self._astype(dtype, copy=copy, errors=errors, values=values, **kwargs)
560
561     def _astype(self, dtype, copy=False, errors="raise", values=None, **kwargs):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/blocks.py in _astype(self, dtype, copy, errors, values, **kwargs)
641                     # _astype_nansafe works fine with 1-d only
642                     vals1d = values.ravel()
--> 643                     values = astype_nansafe(vals1d, dtype, copy=True, **kwargs)
644
645                 # TODO(extension)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/dtypes/cast.py in astype_nansafe(arr, dtype, copy, skipna)
698         if not np.isfinite(arr).all():
699             raise ValueError(
--> 700                 "Cannot convert non-finite values (NA or inf) to " "integer"
701             )
702

ValueError: Cannot convert non-finite values (NA or inf) to integer

# Fixed code
def general_data_processing(
    self, X: DataFrame, X_val: DataFrame, holdout_frac: float, num_bagging_folds: int
):
    """General data processing steps used for all models."""
    X = copy.deepcopy(X)
    # TODO: We should probably uncomment the below lines, NaN label should be treated as just another value in multiclass classification -> We will have to remove missing, compute problem type, and add back missing if multiclass
    # if self.problem_type == MULTICLASS:
    #     X[self.label] = X[self.label].fillna('')

    # Remove all examples with missing labels from this dataset:
    missinglabel_inds = [index for index, x in X[self.label].isna().iteritems() if x]
    if len(missinglabel_inds) > 0:
        logger.warning(
            f"Warning: Ignoring {len(missinglabel_inds)} (out of {len(X)}) training examples for which the label value in column '{self.label}' is missing"
        )
        X = X.drop(missinglabel_inds, axis=0)

    if self.problem_type is None:
        self.problem_type = self.infer_problem_type(X[self.label])

    if X_val is not None and self.label in X_val.columns:
        # TODO: This is not an ideal solution, instead check if bagging and X_val exists with label, then merge them prior to entering general data processing.
        #  This solution should handle virtually all cases correctly, only downside is it might cut more classes than it needs to.
        self.threshold, holdout_frac, num_bagging_folds = (
            self.adjust_threshold_if_necessary(
                X[self.label],
                threshold=self.threshold,
                holdout_frac=1,
                num_bagging_folds=num_bagging_folds,
            )
        )
    else:
        self.threshold, holdout_frac, num_bagging_folds = (
            self.adjust_threshold_if_necessary(
                X[self.label],
                threshold=self.threshold,
                holdout_frac=holdout_frac,
                num_bagging_folds=num_bagging_folds,
            )
        )

    if (
        (self.eval_metric is not None)
        and (self.eval_metric.name in ["log_loss", "pac_score"])
        and (self.problem_type == MULTICLASS)
    ):
        X = augment_rare_classes(X, self.label, self.threshold)

    # Gets labels prior to removal of infrequent classes
    y_uncleaned = X[self.label].copy()

    self.cleaner = Cleaner.construct(
        problem_type=self.problem_type, label=self.label, threshold=self.threshold
    )
    # TODO: What if all classes in X are low frequency in multiclass? Currently we would crash. Not certain how many problems actually have this property
    X = self.cleaner.fit_transform(
        X
    )  # TODO: Consider merging cleaner into label_cleaner
    X, y = self.extract_label(X)
    self.label_cleaner = LabelCleaner.construct(
        problem_type=self.problem_type, y=y, y_uncleaned=y_uncleaned
    )
    y = self.label_cleaner.transform(y)

    if self.label_cleaner.num_classes is not None:
        logger.log(20, f"Train Data Class Count: {self.label_cleaner.num_classes}")

    if X_val is not None and self.label in X_val.columns:
        X_val = self.cleaner.transform(X_val)
        if len(X_val) == 0:
            logger.warning(
                "All X_val data contained low frequency classes, ignoring X_val and generating from subset of X"
            )
            X_val = None
            y_val = None
        else:
            X_val, y_val = self.extract_label(X_val)
            y_val = self.label_cleaner.transform(y_val)
    else:
        y_val = None

    if self.id_columns:
        logger.log(20, f"Dropping ID columns: {self.id_columns}")
        X = X.drop(self.id_columns, axis=1, errors="ignore")
        if X_val is not None:
            X_val = X_val.drop(self.id_columns, axis=1, errors="ignore")

    # TODO: Move this up to top of data before removing data, this way our feature generator is better
    if X_val is not None:
        # Do this if working with SKLearn models, otherwise categorical features may perform very badly on the test set
        logger.log(
            15,
            "Performing general data preprocessing with merged train & validation data, so validation performance may not accurately reflect performance on new test data",
        )
        X_super = pd.concat([X, X_val], ignore_index=True)
        if self.feature_generator.is_fit():
            logger.log(
                20,
                f"{self.feature_generator.__class__.__name__} is already fit, so the training data will be processed via .transform() instead of .fit_transform().",
            )
            X_super = self.feature_generator.transform(X_super)
            self.feature_generator.print_feature_metadata_info()
        else:
            X_super = self.feature_generator.fit_transform(X_super)
        X = X_super.head(len(X)).set_index(X.index)
        X_val = X_super.tail(len(X_val)).set_index(X_val.index)
        del X_super
    else:
        if self.feature_generator.is_fit():
            logger.log(
                20,
                f"{self.feature_generator.__class__.__name__} is already fit, so the training data will be processed via .transform() instead of .fit_transform().",
            )
            X = self.feature_generator.transform(X)
            self.feature_generator.print_feature_metadata_info()
        else:
            X = self.feature_generator.fit_transform(X)

    return X, y, X_val, y_val, holdout_frac, num_bagging_folds