# Buggy code
def _fit(
    self,
    X: DataFrame,
    X_val: DataFrame = None,
    scheduler_options=None,
    hyperparameter_tune=False,
    feature_prune=False,
    holdout_frac=0.1,
    num_bagging_folds=0,
    num_bagging_sets=1,
    stack_ensemble_levels=0,
    hyperparameters=None,
    ag_args_fit=None,
    excluded_model_types=None,
    time_limit=None,
    save_data=False,
    save_bagged_folds=True,
    verbosity=2,
):
    """Arguments:
    X (DataFrame): training data
    X_val (DataFrame): data used for hyperparameter tuning. Note: final model may be trained using this data as well as training data
    hyperparameter_tune (bool): whether to tune hyperparameters or simply use default values
    feature_prune (bool): whether to perform feature selection
    scheduler_options (tuple: (search_strategy, dict): Options for scheduler
    holdout_frac (float): Fraction of data to hold out for evaluating validation performance (ignored if X_val != None, ignored if kfolds != 0)
    num_bagging_folds (int): kfolds used for bagging of models, roughly increases model training time by a factor of k (0: disabled)
    num_bagging_sets (int): number of repeats of kfold bagging to perform (values must be >= 1),
        total number of models trained during bagging = num_bagging_folds * num_bagging_sets
    stack_ensemble_levels : (int) Number of stacking levels to use in ensemble stacking. Roughly increases model training time by factor of stack_levels+1 (0: disabled)
        Default is 0 (disabled). Use values between 1-3 to improve model quality.
        Ignored unless kfolds is also set >= 2
    hyperparameters (dict): keys = hyperparameters + search-spaces for each type of model we should train.
    """
    if hyperparameters is None:
        hyperparameters = {"NN": {}, "GBM": {}}
    # TODO: if provided, feature_types in X, X_val are ignored right now, need to pass to Learner/trainer and update this documentation.
    if time_limit:
        self.time_limit = time_limit
        logger.log(20, f"Beginning AutoGluon training ... Time limit = {time_limit}s")
    else:
        self.time_limit = 1e7
        logger.log(20, "Beginning AutoGluon training ...")
    logger.log(20, f"AutoGluon will save models to {self.path}")
    logger.log(20, f"AutoGluon Version:  {self.version}")
    logger.log(20, f"Train Data Rows:    {len(X)}")
    logger.log(20, f"Train Data Columns: {len(X.columns)}")
    if X_val is not None:
        logger.log(20, f"Tuning Data Rows:    {len(X_val)}")
        logger.log(20, f"Tuning Data Columns: {len(X_val.columns)}")
    time_preprocessing_start = time.time()
    logger.log(20, "Preprocessing data ...")
    X, y, X_val, y_val, holdout_frac, num_bagging_folds = self.general_data_processing(
        X, X_val, holdout_frac, num_bagging_folds
    )
    time_preprocessing_end = time.time()
    self.time_fit_preprocessing = time_preprocessing_end - time_preprocessing_start
    logger.log(
        20,
        f"\tData preprocessing and feature engineering runtime = {round(self.time_fit_preprocessing, 2)}s ...",
    )
    if time_limit:
        time_limit_trainer = time_limit - self.time_fit_preprocessing
    else:
        time_limit_trainer = None

    trainer = self.trainer_type(
        path=self.model_context,
        problem_type=self.label_cleaner.problem_type_transform,
        eval_metric=self.eval_metric,
        stopping_metric=self.stopping_metric,
        num_classes=self.label_cleaner.num_classes,
        feature_types_metadata=self.feature_generator.feature_types_metadata,
        low_memory=True,
        kfolds=num_bagging_folds,
        n_repeats=num_bagging_sets,
        stack_ensemble_levels=stack_ensemble_levels,
        scheduler_options=scheduler_options,
        time_limit=time_limit_trainer,
        save_data=save_data,
        save_bagged_folds=save_bagged_folds,
        random_seed=self.random_seed,
        verbosity=verbosity,
    )

    self.trainer_path = trainer.path
    if self.eval_metric is None:
        self.eval_metric = trainer.eval_metric
    if self.stopping_metric is None:
        self.stopping_metric = trainer.stopping_metric

    self.save()
    trainer.train(
        X,
        y,
        X_val,
        y_val,
        hyperparameter_tune=hyperparameter_tune,
        feature_prune=feature_prune,
        holdout_frac=holdout_frac,
        hyperparameters=hyperparameters,
        ag_args_fit=ag_args_fit,
        excluded_model_types=excluded_model_types,
    )
    self.save_trainer(trainer=trainer)
    time_end = time.time()
    self.time_fit_training = time_end - time_preprocessing_end
    self.time_fit_total = time_end - time_preprocessing_start
    logger.log(
        20,
        f"AutoGluon training complete, total runtime = {round(self.time_fit_total, 2)}s ...",
    )


# Traceback (ValueError)
File delimiter for ../input/house-prices-advanced-regression-techniques/test.csv inferred as ',' (comma). If this is incorrect, please manually load the data as a pandas DataFrame.
Loaded data from: ../input/house-prices-advanced-regression-techniques/test.csv | Columns = 80 / 80 | Rows = 1459 -> 1459
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-48-91455d7b9940> in <module>
1 test_data = task.Dataset(file_path=os.path.join(in_dir, job, 'test.csv'))
2
----> 3 preds = model.predict(test_data)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/task/tabular_prediction/predictor.py in predict(self, dataset, model, as_pandas, use_pred_cache, add_to_pred_cache)
130         """
131         dataset = self.__get_dataset(dataset)
--> 132         return self._learner.predict(X=dataset, model=model, as_pandas=as_pandas, use_pred_cache=use_pred_cache, add_to_pred_cache=add_to_pred_cache)
133
134     def predict_proba(self, dataset, model=None, as_pandas=False, as_multiclass=False):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in predict(self, X, model, as_pandas, use_pred_cache, add_to_pred_cache)
127
128         if len(X_cache_miss) > 0:
--> 129             y_pred_proba = self.predict_proba(X=X_cache_miss, model=model, inverse_transform=False)
130             problem_type = self.trainer_problem_type or self.problem_type
131             y_pred = get_pred_from_proba(y_pred_proba=y_pred_proba, problem_type=problem_type)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in predict_proba(self, X, model, as_pandas, as_multiclass, inverse_transform)
92     # TODO: Add pred_proba_cache functionality as in predict()
93     def predict_proba(self, X: DataFrame, model=None, as_pandas=False, as_multiclass=False, inverse_transform=True):
---> 94         X = self.transform_features(X)
95         y_pred_proba = self.load_trainer().predict_proba(X, model=model)
96         if inverse_transform:

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/ml/learner/abstract_learner.py in transform_features(self, X)
216     def transform_features(self, X):
217         for feature_generator in self.feature_generators:
--> 218             X = feature_generator.transform(X)
219         return X
220

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/utils/decorators.py in inner1(*args, **kwargs)
12         begin = time.time()
13
---> 14         output = func(*args, **kwargs)
15
16         # storing time after function execution

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/autogluon/utils/tabular/features/abstract_feature_generator.py in transform(self, X)
183             raise ValueError(f'Required columns are missing from the provided dataset. Missing columns: {missing_cols}')
184
--> 185         X = X.astype(self.features_init_types)
186         X.reset_index(drop=True, inplace=True)
187         X_features = self.generate_features(X)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/generic.py in astype(self, dtype, copy, errors, **kwargs)
5863                     results.append(
5864                         col.astype(
-> 5865                             dtype=dtype[col_name], copy=copy, errors=errors, **kwargs
5866                         )
5867                     )

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/generic.py in astype(self, dtype, copy, errors, **kwargs)
5880             # else, only a single dtype is given
5881             new_data = self._data.astype(
-> 5882                 dtype=dtype, copy=copy, errors=errors, **kwargs
5883             )
5884             return self._constructor(new_data).__finalize__(self)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/managers.py in astype(self, dtype, **kwargs)
579
580     def astype(self, dtype, **kwargs):
--> 581         return self.apply("astype", dtype=dtype, **kwargs)
582
583     def convert(self, **kwargs):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/managers.py in apply(self, f, axes, filter, do_integrity_check, consolidate, **kwargs)
436                     kwargs[k] = obj.reindex(b_items, axis=axis, copy=align_copy)
437
--> 438             applied = getattr(b, f)(**kwargs)
439             result_blocks = _extend_blocks(applied, result_blocks)
440

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/blocks.py in astype(self, dtype, copy, errors, values, **kwargs)
557
558     def astype(self, dtype, copy=False, errors="raise", values=None, **kwargs):
--> 559         return self._astype(dtype, copy=copy, errors=errors, values=values, **kwargs)
560
561     def _astype(self, dtype, copy=False, errors="raise", values=None, **kwargs):

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/internals/blocks.py in _astype(self, dtype, copy, errors, values, **kwargs)
641                     # _astype_nansafe works fine with 1-d only
642                     vals1d = values.ravel()
--> 643                     values = astype_nansafe(vals1d, dtype, copy=True, **kwargs)
644
645                 # TODO(extension)

~/miniconda3/envs/autogluon/lib/python3.7/site-packages/pandas/core/dtypes/cast.py in astype_nansafe(arr, dtype, copy, skipna)
698         if not np.isfinite(arr).all():
699             raise ValueError(
--> 700                 "Cannot convert non-finite values (NA or inf) to " "integer"
701             )
702

ValueError: Cannot convert non-finite values (NA or inf) to integer

# Fixed code
def _fit(
    self,
    X: DataFrame,
    X_val: DataFrame = None,
    scheduler_options=None,
    hyperparameter_tune=False,
    feature_prune=False,
    holdout_frac=0.1,
    num_bagging_folds=0,
    num_bagging_sets=1,
    stack_ensemble_levels=0,
    hyperparameters=None,
    ag_args_fit=None,
    excluded_model_types=None,
    time_limit=None,
    save_data=False,
    save_bagged_folds=True,
    verbosity=2,
):
    """Arguments:
    X (DataFrame): training data
    X_val (DataFrame): data used for hyperparameter tuning. Note: final model may be trained using this data as well as training data
    hyperparameter_tune (bool): whether to tune hyperparameters or simply use default values
    feature_prune (bool): whether to perform feature selection
    scheduler_options (tuple: (search_strategy, dict): Options for scheduler
    holdout_frac (float): Fraction of data to hold out for evaluating validation performance (ignored if X_val != None, ignored if kfolds != 0)
    num_bagging_folds (int): kfolds used for bagging of models, roughly increases model training time by a factor of k (0: disabled)
    num_bagging_sets (int): number of repeats of kfold bagging to perform (values must be >= 1),
        total number of models trained during bagging = num_bagging_folds * num_bagging_sets
    stack_ensemble_levels : (int) Number of stacking levels to use in ensemble stacking. Roughly increases model training time by factor of stack_levels+1 (0: disabled)
        Default is 0 (disabled). Use values between 1-3 to improve model quality.
        Ignored unless kfolds is also set >= 2
    hyperparameters (dict): keys = hyperparameters + search-spaces for each type of model we should train.
    """
    if hyperparameters is None:
        hyperparameters = {"NN": {}, "GBM": {}}
    # TODO: if provided, feature_types in X, X_val are ignored right now, need to pass to Learner/trainer and update this documentation.
    if time_limit:
        self.time_limit = time_limit
        logger.log(20, f"Beginning AutoGluon training ... Time limit = {time_limit}s")
    else:
        self.time_limit = 1e7
        logger.log(20, "Beginning AutoGluon training ...")
    logger.log(20, f"AutoGluon will save models to {self.path}")
    logger.log(20, f"AutoGluon Version:  {self.version}")
    logger.log(20, f"Train Data Rows:    {len(X)}")
    logger.log(20, f"Train Data Columns: {len(X.columns)}")
    if X_val is not None:
        logger.log(20, f"Tuning Data Rows:    {len(X_val)}")
        logger.log(20, f"Tuning Data Columns: {len(X_val.columns)}")
    time_preprocessing_start = time.time()
    logger.log(20, "Preprocessing data ...")
    X, y, X_val, y_val, holdout_frac, num_bagging_folds = self.general_data_processing(
        X, X_val, holdout_frac, num_bagging_folds
    )
    time_preprocessing_end = time.time()
    self.time_fit_preprocessing = time_preprocessing_end - time_preprocessing_start
    logger.log(
        20,
        f"Data preprocessing and feature engineering runtime = {round(self.time_fit_preprocessing, 2)}s ...",
    )
    if time_limit:
        time_limit_trainer = time_limit - self.time_fit_preprocessing
    else:
        time_limit_trainer = None

    trainer = self.trainer_type(
        path=self.model_context,
        problem_type=self.label_cleaner.problem_type_transform,
        eval_metric=self.eval_metric,
        stopping_metric=self.stopping_metric,
        num_classes=self.label_cleaner.num_classes,
        feature_metadata=self.feature_generator.feature_metadata,
        low_memory=True,
        kfolds=num_bagging_folds,
        n_repeats=num_bagging_sets,
        stack_ensemble_levels=stack_ensemble_levels,
        scheduler_options=scheduler_options,
        time_limit=time_limit_trainer,
        save_data=save_data,
        save_bagged_folds=save_bagged_folds,
        random_seed=self.random_seed,
        verbosity=verbosity,
    )

    self.trainer_path = trainer.path
    if self.eval_metric is None:
        self.eval_metric = trainer.eval_metric
    if self.stopping_metric is None:
        self.stopping_metric = trainer.stopping_metric

    self.save()
    trainer.train(
        X,
        y,
        X_val,
        y_val,
        hyperparameter_tune=hyperparameter_tune,
        feature_prune=feature_prune,
        holdout_frac=holdout_frac,
        hyperparameters=hyperparameters,
        ag_args_fit=ag_args_fit,
        excluded_model_types=excluded_model_types,
    )
    self.save_trainer(trainer=trainer)
    time_end = time.time()
    self.time_fit_training = time_end - time_preprocessing_end
    self.time_fit_total = time_end - time_preprocessing_start
    logger.log(
        20,
        f"AutoGluon training complete, total runtime = {round(self.time_fit_total, 2)}s ...",
    )