jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_run_model_on_fold_regression(self):
        # There aren't any regression tasks on the test server
        openml.config.server = self.production_server
        task = openml.tasks.get_task(2999)

        X, y = task.get_X_and_y()
        train_indices, test_indices = task.get_train_test_split_indices(repeat=0, fold=0, sample=0)
        X_train = X[train_indices]
        y_train = y[train_indices]
        X_test = X[test_indices]
        y_test = y[test_indices]

        pipeline = sklearn.pipeline.Pipeline(
            steps=[("imp", sklearn.impute.SimpleImputer()), ("clf", sklearn.tree.DecisionTreeRegressor())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == y_test.shape
        assert y_hat_proba is None

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_failed_serialization_of_custom_class(self):
        """Check if any custom class inherited from sklearn expectedly fails serialization"""
        try:
            from sklearn.impute import SimpleImputer
        except ImportError:
            # for lower versions
            from sklearn.preprocessing import Imputer as SimpleImputer

        import sklearn.tree
        from sklearn.compose import ColumnTransformer
        from sklearn.pipeline import Pipeline, make_pipeline
        from sklearn.preprocessing import OneHotEncoder, StandardScaler

        cat_imp = make_pipeline(
            SimpleImputer(strategy="most_frequent"),
            OneHotEncoder(handle_unknown="ignore"),
        )
        cont_imp = make_pipeline(CustomImputer(), StandardScaler())
        ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
        clf = Pipeline(
            steps=[("preprocess", ct), ("estimator", sklearn.tree.DecisionTreeClassifier())],
        )  # build a sklearn classifier

        task = openml.tasks.get_task(253)  # profb; crossvalidation
        try:
            _ = openml.runs.run_model_on_task(clf, task)
        except AttributeError as e:
            if e.args[0] == "module '__main__' has no attribute '__version__'":
                raise AttributeError(e)
            else:
                raise Exception(e)
//  [python]
def test_failed_serialization_of_custom_class(self):
        """Check if any custom class inherited from sklearn expectedly fails serialization"""
        from sklearn.impute import SimpleImputer

        import sklearn.tree
        from sklearn.compose import ColumnTransformer
        from sklearn.pipeline import Pipeline, make_pipeline
        from sklearn.preprocessing import OneHotEncoder, StandardScaler

        cat_imp = make_pipeline(
            SimpleImputer(strategy="most_frequent"),
            OneHotEncoder(handle_unknown="ignore"),
        )
        cont_imp = make_pipeline(CustomImputer(), StandardScaler())
        ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
        clf = Pipeline(
            steps=[("preprocess", ct), ("estimator", sklearn.tree.DecisionTreeClassifier())],
        )  # build a sklearn classifier

        task = openml.tasks.get_task(253)  # profb; crossvalidation
        try:
            _ = openml.runs.run_model_on_task(clf, task)
        except AttributeError as e:
            if e.args[0] == "module '__main__' has no attribute '__version__'":
                raise AttributeError(e)
            else:
                raise Exception(e)
//  [python]
def test_run_model_on_fold_classification_1_dataframe(self):
        from sklearn.compose import ColumnTransformer

        task = openml.tasks.get_task(1)  # anneal; crossvalidation

        # diff test_run_model_on_fold_classification_1_array()
        X, y = task.get_X_and_y(dataset_format="dataframe")
        train_indices, test_indices = task.get_train_test_split_indices(repeat=0, fold=0, sample=0)
        X_train = X.iloc[train_indices]
        y_train = y.iloc[train_indices]
        X_test = X.iloc[test_indices]
        y_test = y.iloc[test_indices]

        # Helper functions to return required columns for ColumnTransformer
        cat_imp = make_pipeline(
            SimpleImputer(strategy="most_frequent"),
            OneHotEncoder(handle_unknown="ignore", sparse=False),
        )
        cont_imp = make_pipeline(CustomImputer(strategy="mean"), StandardScaler())
        ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
        pipeline = sklearn.pipeline.Pipeline(
            steps=[("transform", ct), ("estimator", sklearn.tree.DecisionTreeClassifier())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == y_test.shape
        assert isinstance(y_hat_proba, pd.DataFrame)
        assert y_hat_proba.shape == (y_test.shape[0], 6)
        np.testing.assert_array_almost_equal(np.sum(y_hat_proba, axis=1), np.ones(y_test.shape))
        # The class '4' (at index 3) is not present in the training data. We check that the
        # predicted probabilities for that class are zero!
        np.testing.assert_array_almost_equal(
            y_hat_proba.iloc[:, 3].to_numpy(),
            np.zeros(y_test.shape),
        )
        for i in (0, 1, 2, 4, 5):
            assert np.any(y_hat_proba.iloc[:, i].to_numpy() != np.zeros(y_test.shape))

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_run_model_on_fold_classification_1_dataframe(self):
        from sklearn.compose import ColumnTransformer

        task = openml.tasks.get_task(1)  # anneal; crossvalidation

        # diff test_run_model_on_fold_classification_1_array()
        X, y = task.get_X_and_y(dataset_format="dataframe")
        train_indices, test_indices = task.get_train_test_split_indices(repeat=0, fold=0, sample=0)
        X_train = X.iloc[train_indices]
        y_train = y.iloc[train_indices]
        X_test = X.iloc[test_indices]
        y_test = y.iloc[test_indices]

        # Helper functions to return required columns for ColumnTransformer
        cat_imp = make_pipeline(
            SimpleImputer(strategy="most_frequent"),
            OneHotEncoder(handle_unknown="ignore", sparse=False),
        )
        cont_imp = make_pipeline(SimpleImputer(strategy="mean"), StandardScaler())
        ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
        pipeline = sklearn.pipeline.Pipeline(
            steps=[("transform", ct), ("estimator", sklearn.tree.DecisionTreeClassifier())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == y_test.shape
        assert isinstance(y_hat_proba, pd.DataFrame)
        assert y_hat_proba.shape == (y_test.shape[0], 6)
        np.testing.assert_array_almost_equal(np.sum(y_hat_proba, axis=1), np.ones(y_test.shape))
        # The class '4' (at index 3) is not present in the training data. We check that the
        # predicted probabilities for that class are zero!
        np.testing.assert_array_almost_equal(
            y_hat_proba.iloc[:, 3].to_numpy(),
            np.zeros(y_test.shape),
        )
        for i in (0, 1, 2, 4, 5):
            assert np.any(y_hat_proba.iloc[:, i].to_numpy() != np.zeros(y_test.shape))

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_run_model_on_fold_classification_3(self):
        class HardNaiveBayes(sklearn.naive_bayes.GaussianNB):
            # class for testing a naive bayes classifier that does not allow soft
            # predictions
            def predict_proba(*args, **kwargs):
                raise AttributeError("predict_proba is not available when " "probability=False")

        # task 1 (test server) is important: it is a task with an unused class
        tasks = [
            1,  # anneal; crossvalidation
            3,  # anneal; crossvalidation
            115,  # diabetes; crossvalidation
        ]
        flow = unittest.mock.Mock()
        flow.name = "dummy"

        for task_id in tasks:
            task = openml.tasks.get_task(task_id)
            X, y = task.get_X_and_y()
            train_indices, test_indices = task.get_train_test_split_indices(
                repeat=0,
                fold=0,
                sample=0,
            )
            X_train = X[train_indices]
            y_train = y[train_indices]
            X_test = X[test_indices]
            clf1 = sklearn.pipeline.Pipeline(
                steps=[
                    ("imputer", SimpleImputer()),
                    ("estimator", sklearn.naive_bayes.GaussianNB()),
                ],
            )
            clf2 = sklearn.pipeline.Pipeline(
                steps=[("imputer", SimpleImputer()), ("estimator", HardNaiveBayes())],
            )

            pred_1, proba_1, _, _ = self.extension._run_model_on_fold(
                model=clf1,
                task=task,
                X_train=X_train,
                y_train=y_train,
                X_test=X_test,
                fold_no=0,
                rep_no=0,
            )
            pred_2, proba_2, _, _ = self.extension._run_model_on_fold(
                model=clf2,
                task=task,
                X_train=X_train,
                y_train=y_train,
                X_test=X_test,
                fold_no=0,
                rep_no=0,
            )

            # verifies that the predictions are identical
            np.testing.assert_array_equal(pred_1, pred_2)
            np.testing.assert_array_almost_equal(np.sum(proba_1, axis=1), np.ones(X_test.shape[0]))
            # Test that there are predictions other than ones and zeros
            assert np.sum(proba_1.to_numpy() == 0) + np.sum(proba_1.to_numpy() == 1) < X_test.shape[
                0
            ] * len(task.class_labels)

            np.testing.assert_array_almost_equal(np.sum(proba_2, axis=1), np.ones(X_test.shape[0]))
            # Test that there are only ones and zeros predicted
            assert np.sum(proba_2.to_numpy() == 0) + np.sum(
                proba_2.to_numpy() == 1
            ) == X_test.shape[0] * len(task.class_labels)
//  [python]
def imputer_transform(data):
    imputer = Imputer()
    imputer.fit(data)
    return imputer.transform(data)
//  [python]
def imputer_transform(data):
    imputer = SimpleImputer()
    imputer.fit(data)
    return imputer.transform(data)
//  [python]
def load_datasets(feature_paths, label_paths):
    # 读取特征文件列表和标签文件列表中的内容，归并后返回
    feature = np.ndarray(shape=(0, 41))
    # 定义feature数组变量，列和特征维度一致为41
    label = np.ndarray(shape=(0, 1))
    # 定义空的标签变量，列数与签维度一致为1
    for file in feature_paths:
        # 使用pandas库的read_table函数读取一个特征文件的内容
        # 其中指定分隔符为逗号、缺失值为问号且不包含表头
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        # 使用Imputer函数，通过设定strategy参数为'mean'
        # 使用平均值补全缺失值
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        # fit()函数用于训练预处理器
        imp.fit(df)
        # transform()函数用于生成预处理结果
        df = imp.transform(df)
        # 将预处理后的数据加入特征集feature
        feature = np.concatenate((feature, df))
    # 依次遍历完所有特征文件
    for file in label_paths:
        # 调用pandas库中的read_table函数读取一个标签文件的内容
        # 指定分隔符为逗号且不包含表头行
        df = pd.read_table(file, header=None)
        # 由于标签文件没有缺失值，所以直接将读取到的新数据加入labe集合
        label = np.concatenate((label, df))
    # 将标签规整为一维向量
    label = np.ravel(label)
    return feature, label
//  [python]
def load_datasets(feature_paths, label_paths):
    # 读取特征文件列表和标签文件列表中的内容，归并后返回
    feature = np.ndarray(shape=(0, 41))
    # 定义feature数组变量，列和特征维度一致为41
    label = np.ndarray(shape=(0, 1))
    # 定义空的标签变量，列数与签维度一致为1
    for file in feature_paths:
        # 使用pandas库的read_table函数读取一个特征文件的内容
        # 其中指定分隔符为逗号、缺失值为问号且不包含表头
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        # 使用Imputer函数，通过设定strategy参数为'mean'
        # 使用平均值补全缺失值
        imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        # fit()函数用于训练预处理器
        imp.fit(df)
        # transform()函数用于生成预处理结果
        df = imp.transform(df)
        # 将预处理后的数据加入特征集feature
        feature = np.concatenate((feature, df))
    # 依次遍历完所有特征文件
    for file in label_paths:
        # 调用pandas库中的read_table函数读取一个标签文件的内容
        # 指定分隔符为逗号且不包含表头行
        df = pd.read_table(file, header=None)
        # 由于标签文件没有缺失值，所以直接将读取到的新数据加入labe集合
        label = np.concatenate((label, df))
    # 将标签规整为一维向量
    label = np.ravel(label)
    return feature, label
//  [python]
def load_datasets(feature_paths, label_paths):
    feature = np.ndarray(shape=(0,41))
    label = np.ndarray(shape=(0,1))
    for file in feature_paths:
        # 使用pandas库的read_table函数读取一个特征文件内容
        # 指定分隔符为逗号 缺失值为问号 文件中不包含表头行
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        # 使用Imputer函数,通过设定strategy参数为'mean'
        # 使用平均值对缺失数据补全,fit()函数用于训练预处理器,
        # transform()函数用于生成预处理结果
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        imp.fit(df)
        df = imp.transform(df)
        # 将预处理后的数据加入feature,依次遍历完所有特征文件
        feature = np.concatenate((feature, df))
     
    for file in label_paths:
        # 同上
        df = pd.read_table(file, header=None)
        # 标签文件没有缺失值,所以直接将读取到的新数据加入label集合
        label = np.concatenate((label, df))
         
    label = np.ravel(label)
    # 将特征集合feature与标签集合label返回
    return feature, label
//  [python]
def load_datasets(feature_paths, label_paths):
    feature = np.ndarray(shape=(0,41))
    label = np.ndarray(shape=(0,1))
    for file in feature_paths:
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        df = imp.fit_transform(df)
        feature = np.concatenate((feature, df))
     
    for file in label_paths:
        df = pd.read_table(file, header=None)
        label = np.concatenate((label, df))
         
    label = np.ravel(label)
    return feature, label
//  [python]
def impute_and_scale(df, scaling='std', imputing='mean', dropna='all'):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == 'none':
        mat = df.values
    else:
        imputer = Imputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == 'none':
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == 'maxabs':
        scaler = MaxAbsScaler()
    elif scaling == 'minmax':
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling='std', imputing='mean', dropna='all'):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == 'none':
        mat = df.values
    else:
        imputer = SimpleImputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == 'none':
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == 'maxabs':
        scaler = MaxAbsScaler()
    elif scaling == 'minmax':
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        imputer = Imputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        imputer = SimpleImputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def test_fill_continuous(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.fill(continuous=['Age'], strategy='mean')
    hage = hdf_filled.cols['Age'][:].values

    imputer = Imputer(strategy='mean').fit(pdf[['Age']])
    pdf_filled = imputer.transform(pdf[['Age']])
    age = pdf_filled.ravel()

    npt.assert_array_equal(hage, age)
    npt.assert_array_equal(hdf_filled.statistics_['Age'], imputer.statistics_[0])
//  [python]
def test_fill_continuous(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.fill(continuous=['Age'], strategy='mean')
    hage = hdf_filled.cols['Age'][:].values

    imputer = SimpleImputer(strategy='mean').fit(pdf[['Age']])
    pdf_filled = imputer.transform(pdf[['Age']])
    age = pdf_filled.ravel()

    npt.assert_array_equal(hage, age)
    npt.assert_array_equal(hdf_filled.statistics_['Age'], imputer.statistics_[0])
//  [python]
def featureSet(data):
    imputer = Imputer(missing_values='NaN', strategy='mean', axis=1)
    imputer.fit(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])
    x_new = imputer.transform(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])

    data_num = len(data)
    XList = []
    for row in range(0, data_num):
        tmp_list = []
        tmp_list.append(data.iloc[row]['Oddset'])
        tmp_list.append(data.iloc[row]['li'])
        tmp_list.append(data.iloc[row]['bet365'])
        tmp_list.append(data.iloc[row]['interw'])
        tmp_list.append(data.iloc[row]['wl'])
        tmp_list.append(data.iloc[row]['w'])
        tmp_list.append(data.iloc[row]['ao'])
        # tmp_list.append(data.iloc[row]['10bet'])
        # tmp_list.append(data.iloc[row]['jbb'])
        # tmp_list.append(data.iloc[row]['ms'])
        # tmp_list.append(data.iloc[row]['ysb'])
        tmp_list.append(x_new[row][0])
        tmp_list.append(x_new[row][1])
        tmp_list.append(x_new[row][2])
        tmp_list.append(x_new[row][3])
        tmp_list.append(x_new[row][4])
        tmp_list.append(x_new[row][5])
        XList.append(tmp_list)
    yList = data.y.values
    return XList, yList
//  [python]
def featureSet(data):
    imputer = SimpleImputer(missing_values='NaN', strategy='mean')
    imputer.fit(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])
    x_new = imputer.transform(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])

    data_num = len(data)
    XList = []
    for row in range(0, data_num):
        tmp_list = []
        tmp_list.append(data.iloc[row]['Oddset'])
        tmp_list.append(data.iloc[row]['li'])
        tmp_list.append(data.iloc[row]['bet365'])
        tmp_list.append(data.iloc[row]['interw'])
        tmp_list.append(data.iloc[row]['wl'])
        tmp_list.append(data.iloc[row]['w'])
        tmp_list.append(data.iloc[row]['ao'])
        # tmp_list.append(data.iloc[row]['10bet'])
        # tmp_list.append(data.iloc[row]['jbb'])
        # tmp_list.append(data.iloc[row]['ms'])
        # tmp_list.append(data.iloc[row]['ysb'])
        tmp_list.append(x_new[row][0])
        tmp_list.append(x_new[row][1])
        tmp_list.append(x_new[row][2])
        tmp_list.append(x_new[row][3])
        tmp_list.append(x_new[row][4])
        tmp_list.append(x_new[row][5])
        XList.append(tmp_list)
    yList = data.y.values
    return XList, yList
//  [python]
def test_imputer_no_nan_inputs(self):
        model = Imputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_imputer(model, data)
//  [python]
def test_imputer_no_nan_inputs(self):
        model = SimpleImputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_imputer(model, data)
//  [python]
def loadTestData(filePath):
    data = pd.read_csv(filepath_or_buffer=filePath)
    imputer = Imputer(missing_values='NaN', strategy='mean', axis=1)
    imputer.fit(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])
    x_new = imputer.transform(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])

    data_num = len(data)
    XList = []
    for row in range(0, data_num):
        tmp_list = []
        tmp_list.append(data.iloc[row]['Oddset'])
        tmp_list.append(data.iloc[row]['li'])
        tmp_list.append(data.iloc[row]['bet365'])
        tmp_list.append(data.iloc[row]['interw'])
        tmp_list.append(data.iloc[row]['wl'])
        tmp_list.append(data.iloc[row]['w'])
        tmp_list.append(data.iloc[row]['ao'])
        # tmp_list.append(data.iloc[row]['10bet'])
        # tmp_list.append(data.iloc[row]['jbb'])
        # tmp_list.append(data.iloc[row]['ms'])
        # tmp_list.append(data.iloc[row]['ysb'])
        tmp_list.append(x_new[row][0])
        tmp_list.append(x_new[row][1])
        tmp_list.append(x_new[row][2])
        tmp_list.append(x_new[row][3])
        tmp_list.append(x_new[row][4])
        tmp_list.append(x_new[row][5])
        XList.append(tmp_list)
    return XList
//  [python]
def loadTestData(filePath):
    data = pd.read_csv(filepath_or_buffer=filePath)
    imputer = SimpleImputer(missing_values='NaN', strategy='mean')
    imputer.fit(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])
    x_new = imputer.transform(data.loc[:, ['10bet', 'jbb', 'ms', 'ysb', 'Pinnacle', 'SNAI']])

    data_num = len(data)
    XList = []
    for row in range(0, data_num):
        tmp_list = []
        tmp_list.append(data.iloc[row]['Oddset'])
        tmp_list.append(data.iloc[row]['li'])
        tmp_list.append(data.iloc[row]['bet365'])
        tmp_list.append(data.iloc[row]['interw'])
        tmp_list.append(data.iloc[row]['wl'])
        tmp_list.append(data.iloc[row]['w'])
        tmp_list.append(data.iloc[row]['ao'])
        # tmp_list.append(data.iloc[row]['10bet'])
        # tmp_list.append(data.iloc[row]['jbb'])
        # tmp_list.append(data.iloc[row]['ms'])
        # tmp_list.append(data.iloc[row]['ysb'])
        tmp_list.append(x_new[row][0])
        tmp_list.append(x_new[row][1])
        tmp_list.append(x_new[row][2])
        tmp_list.append(x_new[row][3])
        tmp_list.append(x_new[row][4])
        tmp_list.append(x_new[row][5])
        XList.append(tmp_list)
    return XList
//  [python]
def test_stratify_fill(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.stratify(['Pclass']).fill(continuous=['Age'], categorical=['Embarked'])
    hage = hdf_filled.orderBy('Pclass').cols['Age'][:].values
    hembarked = hdf_filled.orderBy('PassengerId').cols['Embarked'][:].values

    pdf_filled = []
    statistics = {'Age': {}}
    for pclass in [1, 2, 3]:
        filtered = pdf.query('Pclass == {}'.format(pclass))[['Age']]
        imputer = Imputer(strategy='mean').fit(filtered)
        pdf_filled.append(imputer.transform(filtered))
        statistics['Age'].update({'Pclass == "{}"'.format(pclass): imputer.statistics_[0]})
    pdf_filled = np.concatenate(pdf_filled, axis=0)
    age = pdf_filled.ravel()

    npt.assert_array_equal(hage, age)
    npt.assert_array_equal(hembarked, pdf.fillna({'Embarked': 'S'}).sort_values(by='PassengerId')['Embarked'].values)
    npt.assert_array_equal(sorted(list(hdf_filled.statistics_['Age'])),
                           sorted(list(statistics['Age'])))
//  [python]
def test_stratify_fill(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.stratify(['Pclass']).fill(continuous=['Age'], categorical=['Embarked'])
    hage = hdf_filled.orderBy('Pclass').cols['Age'][:].values
    hembarked = hdf_filled.orderBy('PassengerId').cols['Embarked'][:].values

    pdf_filled = []
    statistics = {'Age': {}}
    for pclass in [1, 2, 3]:
        filtered = pdf.query('Pclass == {}'.format(pclass))[['Age']]
        imputer = SimpleImputer(strategy='mean').fit(filtered)
        pdf_filled.append(imputer.transform(filtered))
        statistics['Age'].update({'Pclass == "{}"'.format(pclass): imputer.statistics_[0]})
    pdf_filled = np.concatenate(pdf_filled, axis=0)
    age = pdf_filled.ravel()

    npt.assert_array_equal(hage, age)
    npt.assert_array_equal(hembarked, pdf.fillna({'Embarked': 'S'}).sort_values(by='PassengerId')['Embarked'].values)
    npt.assert_array_equal(sorted(list(hdf_filled.statistics_['Age'])),
                           sorted(list(statistics['Age'])))
//  [python]
def test_conversion_bad_inputs(self):
        # Error on converting an untrained model
        with self.assertRaises(Exception):
            model = Imputer()
            spec = converter.convert(model, "data", "out")

        # Check the expected class during conversion.
        with self.assertRaises(Exception):
            from sklearn.linear_model import LinearRegression

            model = LinearRegression()
            spec = converter.convert(model, "data", "out")
//  [python]
def test_conversion_bad_inputs(self):
        # Error on converting an untrained model
        with self.assertRaises(Exception):
            model = SimpleImputer()
            spec = converter.convert(model, "data", "out")

        # Check the expected class during conversion.
        with self.assertRaises(Exception):
            from sklearn.linear_model import LinearRegression

            model = LinearRegression()
            spec = converter.convert(model, "data", "out")
//  [python]
def evaluate(train, train_labels, test, test_labels):
    """Evaluate a training dataset with a standard sklearn model"""
    
    # Use the same model for each training set for now
    model = RandomForestRegressor(n_estimators = 100, 
                                  random_state = 50, n_jobs = -1)
    
    train = train.replace({np.inf: np.nan})
    test = test.replace({np.inf: np.nan})
    
    feature_names = list(train.columns)
    
    # Impute the missing values
    imputer = Imputer(strategy = 'median', axis = 1)
    train = imputer.fit_transform(train)
    test = imputer.transform(test)
    
    # Metric     
    mape_scorer = make_scorer(mape, greater_is_better = False)
    
    # Fit on the training data and make predictions
    model.fit(train, train_labels)
    preds = model.predict(test)
    
    cv_score = -1 * cross_val_score(model, train, train_labels, 
                                    scoring = mape_scorer, cv = 5)
    
    # Calculate the performance
    mape_score = mape(test_labels, preds)
    print('5-fold CV MAPE: {:.2f} with std: {:.2f}'.format(cv_score.mean(),cv_score.std()))
    print('Test MAPE: {:.2f}.'.format(mape_score))
    feature_importances = pd.DataFrame({'feature': feature_names, 
                                        'importance': model.feature_importances_})
    
    return preds, feature_importances
//  [python]
def evaluate(train, train_labels, test, test_labels):
    """Evaluate a training dataset with a standard sklearn model"""
    
    # Use the same model for each training set for now
    model = RandomForestRegressor(n_estimators = 100, 
                                  random_state = 50, n_jobs = -1)
    
    train = train.replace({np.inf: np.nan})
    test = test.replace({np.inf: np.nan})
    
    feature_names = list(train.columns)
    
    # Impute the missing values
    imputer = SimpleImputer(strategy = 'median')
    train = imputer.fit_transform(train)
    test = imputer.transform(test)
    
    # Metric     
    mape_scorer = make_scorer(mape, greater_is_better = False)
    
    # Fit on the training data and make predictions
    model.fit(train, train_labels)
    preds = model.predict(test)
    
    cv_score = -1 * cross_val_score(model, train, train_labels, 
                                    scoring = mape_scorer, cv = 5)
    
    # Calculate the performance
    mape_score = mape(test_labels, preds)
    print('5-fold CV MAPE: {:.2f} with std: {:.2f}'.format(cv_score.mean(),cv_score.std()))
    print('Test MAPE: {:.2f}.'.format(mape_score))
    feature_importances = pd.DataFrame({'feature': feature_names, 
                                        'importance': model.feature_importances_})
    
    return preds, feature_importances
//  [python]
def setUpClass(self):
        """
        Set up the unit test by loading the dataset and training a model.
        """
        from sklearn.datasets import load_boston

        scikit_data = load_boston()
        # axis parameter deprecated in SimpleImputer >= 0.22. which now imputes
        # only along columns as desired here.
        if _SKLEARN_VERSION >= StrictVersion("0.22"):
            scikit_model = Imputer(strategy="most_frequent")
        else:
            scikit_model = Imputer(strategy="most_frequent", axis=0)
        scikit_data["data"][1, 8] = np.NaN

        input_data = scikit_data["data"][:, 8].reshape(-1, 1)
        scikit_model.fit(input_data, scikit_data["target"])

        # Save the data and the model
        self.scikit_data = scikit_data
        self.scikit_model = scikit_model
//  [python]
def setUpClass(self):
        """
        Set up the unit test by loading the dataset and training a model.
        """
        from sklearn.datasets import load_boston
        from sklearn.impute import SimpleImputer

        scikit_data = load_boston()
        scikit_model = SimpleImputer(strategy="most_frequent")
        scikit_data["data"][1, 8] = np.NaN

        input_data = scikit_data["data"][:, 8].reshape(-1, 1)
        scikit_model.fit(input_data, scikit_data["target"])

        # Save the data and the model
        self.scikit_data = scikit_data
        self.scikit_model = scikit_model
//  [python]
def test_pipeline_column_transformer_pipeline_imputer_scaler_lr(self):
        X = np.array([[1, 2], [3, np.nan], [3, 0]], dtype=np.float32)
        y = np.array([1, 0, 1])
        model = Pipeline(
            [
                (
                    "ct",
                    ColumnTransformer(
                        [
                            (
                                "pipeline1",
                                Pipeline(
                                    [
                                        ("imputer", SimpleImputer()),
                                        ("scaler", StandardScaler()),
                                    ]
                                ),
                                [0],
                            ),
                            (
                                "pipeline2",
                                Pipeline(
                                    [
                                        ("imputer", SimpleImputer()),
                                        ("scaler", RobustScaler()),
                                    ]
                                ),
                                [1],
                            ),
                        ]
                    ),
                ),
                ("lr", LogisticRegression(solver="liblinear")),
            ]
        )
        model.fit(X, y)
        model_onnx = convert_sklearn(
            model,
            "pipelinewithinpipeline",
            [("input", FloatTensorType([None, X.shape[1]]))],
            target_opset=TARGET_OPSET,
        )
        self.assertTrue(model_onnx is not None)
        dump_data_and_model(
            X, model, model_onnx, basename="SklearnPipelineCTPipelineImputerScalerLR"
        )
//  [python]
def random_search(train, train_labels):
    """Random Search on a set of features using a random forest"""
    
    # Metric     
    mape_scorer = make_scorer(mape, greater_is_better = False)
    scorer = mape_scorer

    param_grid = {
        'n_estimators': [int(x) for x in np.linspace(50, 1000, num = 100)],
        'max_depth': [None] + [int(x) for x in np.linspace(4, 20)],
        'min_samples_leaf': [1, 2, 5, 10],
        'max_features': ['auto', 'sqrt', 0.5, 'log2', None]
    }
    
    # Impute the missing values
    imputer = Imputer(strategy = 'median', axis = 1)
    train = imputer.fit_transform(train.replace({np.inf:np.nan}))
    
    # Make a model 
    reg = RandomForestRegressor(n_jobs = -1, random_state = 50)

    # RandomizedSearchCV object
    random = RandomizedSearchCV(reg, param_grid, n_iter = 100, scoring = scorer, 
                                verbose = 1, n_jobs = -1, cv = 5, random_state = 50)

    # Fit on the training data
    random.fit(train, train_labels)
    
    best_score = -1 * random.best_score_
    best_score_std = random.cv_results_['std_test_score'][np.argmax(random.cv_results_['mean_test_score'])]
    best_model = random.best_estimator_
    
    print('Best 5-fold CV Score: {:.2f} with std: {:.2f}.'.format(best_score, best_score_std))
    print('\nBest Hyperparameters:')
    print(random.best_params_)
    
    return best_model
//  [python]
def random_search(train, train_labels):
    """Random Search on a set of features using a random forest"""
    
    # Metric     
    mape_scorer = make_scorer(mape, greater_is_better = False)
    scorer = mape_scorer

    param_grid = {
        'n_estimators': [int(x) for x in np.linspace(50, 1000, num = 100)],
        'max_depth': [None] + [int(x) for x in np.linspace(4, 20)],
        'min_samples_leaf': [1, 2, 5, 10],
        'max_features': ['auto', 'sqrt', 0.5, 'log2', None]
    }
    
    # Impute the missing values
    imputer = SimpleImputer(strategy = 'median')
    train = imputer.fit_transform(train.replace({np.inf:np.nan}))
    
    # Make a model 
    reg = RandomForestRegressor(n_jobs = -1, random_state = 50)

    # RandomizedSearchCV object
    random = RandomizedSearchCV(reg, param_grid, n_iter = 100, scoring = scorer, 
                                verbose = 1, n_jobs = -1, cv = 5, random_state = 50)

    # Fit on the training data
    random.fit(train, train_labels)
    
    best_score = -1 * random.best_score_
    best_score_std = random.cv_results_['std_test_score'][np.argmax(random.cv_results_['mean_test_score'])]
    best_model = random.best_estimator_
    
    print('Best 5-fold CV Score: {:.2f} with std: {:.2f}.'.format(best_score, best_score_std))
    print('\nBest Hyperparameters:')
    print(random.best_params_)
    
    return best_model
//  [python]
def read_input(test_size=TEST_FRACTION_SIZE):
    """Read input data and split it into train and test."""
    download(TRAINING_URL, TRAINING_FILE)
    data = pd.read_csv(TRAINING_FILE)
    data.dropna(axis=0, subset=['SalePrice'], inplace=True)

    y = data.SalePrice
    X = data.drop(['SalePrice'], axis=1).select_dtypes(exclude=['object'])

    train_X, test_X, train_y, test_y = train_test_split(X.values,
                                                        y.values,
                                                        test_size=test_size,
                                                        shuffle=False)

    imputer = Imputer()
    train_X = imputer.fit_transform(train_X)
    test_X = imputer.transform(test_X)

    return (train_X, train_y), (test_X, test_y)
//  [python]
def read_input(test_size=TEST_FRACTION_SIZE):
    """Read input data and split it into train and test."""
    download(TRAINING_URL, TRAINING_FILE)
    data = pd.read_csv(TRAINING_FILE)
    data.dropna(axis=0, subset=['SalePrice'], inplace=True)

    y = data.SalePrice
    X = data.drop(['SalePrice'], axis=1).select_dtypes(exclude=['object'])

    train_X, test_X, train_y, test_y = train_test_split(X.values,
                                                        y.values,
                                                        test_size=test_size,
                                                        shuffle=False)

    imputer = SimpleImputer()
    train_X = imputer.fit_transform(train_X)
    test_X = imputer.transform(test_X)

    return (train_X, train_y), (test_X, test_y)
//  [python]
def SVM_training(Training,label,SVM_param,cv,threads=6):## 分类方法：SVM
     
    SVM_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                    ('scl',StandardScaler()),
                    ('svc',SVC(kernel="linear",random_state= 1,probability=True))])
    
    SVM_gs = GridSearchCV(estimator=SVM_pipe,
                 param_grid=SVM_param,
                 scoring='roc_auc',
                 cv=cv,
                 n_jobs=threads)

    SVM_gs.fit(Training,label)
     
    SVM_pipe.set_params(**SVM_gs.best_params_)

    SVM_pipe.fit(Training,label)
    
    return(SVM_pipe)
//  [python]
def SVM_training(Training,label,SVM_param,cv,threads=6):## 分类方法：SVM
     
    SVM_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                    ('scl',StandardScaler()),
                    ('svc',SVC(kernel="linear",random_state= 1,probability=True))])
    
    SVM_gs = GridSearchCV(estimator=SVM_pipe,
                 param_grid=SVM_param,
                 scoring='roc_auc',
                 cv=cv,
                 n_jobs=threads)

    SVM_gs.fit(Training,label)
     
    SVM_pipe.set_params(**SVM_gs.best_params_)

    SVM_pipe.fit(Training,label)
    
    return(SVM_pipe)
//  [python]
def RandomForest_training(Training,label,RandomForest_param,cv,threads=6):## 分类方法：随机森林

    RandomForest_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                              ('rfc',RandomForestClassifier())])
    
    RandomForest_gs = GridSearchCV(estimator=RandomForest_pipe,
                 param_grid=RandomForest_param,
                 scoring='roc_auc',
                 cv=cv,
                 n_jobs=threads)

    RandomForest_gs.fit(Training,label)
   
    RandomForest_pipe.set_params(**RandomForest_gs.best_params_)
    
    RandomForest_pipe.fit(Training,label)
    
    return(RandomForest_pipe)
//  [python]
def RandomForest_training(Training,label,RandomForest_param,cv,threads=6):## 分类方法：随机森林

    RandomForest_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                              ('rfc',RandomForestClassifier())])
    
    RandomForest_gs = GridSearchCV(estimator=RandomForest_pipe,
                 param_grid=RandomForest_param,
                 scoring='roc_auc',
                 cv=cv,
                 n_jobs=threads)

    RandomForest_gs.fit(Training,label)
   
    RandomForest_pipe.set_params(**RandomForest_gs.best_params_)
    
    RandomForest_pipe.fit(Training,label)
    
    return(RandomForest_pipe)
//  [python]
def RandomForest_training_single(Training,label,RandomForest_param):

    RandomForest_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                              ('rfc',RandomForestClassifier())])
   
    RandomForest_pipe.set_params(**RandomForest_param)
    
    RandomForest_pipe.fit(Training,label)
    
    return(RandomForest_pipe)
//  [python]
def RandomForest_training_single(Training,label,RandomForest_param):

    RandomForest_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                              ('rfc',RandomForestClassifier())])
   
    RandomForest_pipe.set_params(**RandomForest_param)
    
    RandomForest_pipe.fit(Training,label)
    
    return(RandomForest_pipe)
//  [python]
def SVM_training_single(Training,label,SVM_param):
     
    SVM_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                    ('scl',StandardScaler()),
                    ('svc',SVC(kernel="linear",random_state= 1,probability=True))])
     
    SVM_pipe.set_params(**SVM_param)

    SVM_pipe.fit(Training,label)
    
    return(SVM_pipe)
//  [python]
def SVM_training_single(Training,label,SVM_param):
     
    SVM_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                    ('scl',StandardScaler()),
                    ('svc',SVC(kernel="linear",random_state= 1,probability=True))])
     
    SVM_pipe.set_params(**SVM_param)

    SVM_pipe.fit(Training,label)
    
    return(SVM_pipe)
//  [python]
def build_estimator(config):
    """
    Build the estimator object from the parameters in config.

    :param config:
    :return:
    """
    regressor = config['Model']['regressor']
    sklearn_kwargs = config['Model']['Parameters']
    train_individual = config['Model']['train_individual']
    ada_boost = config['Model'].get('Ada boosting', None)
    rain_tuning = config['Model'].get('Rain tuning', None)
    bootstrap = config['Model'].get('Bootstrapping', None)
    Regressor = get_object('sklearn.%s' % regressor)
    if config['verbose']:
        print('build_estimator: using sklearn.%s as estimator...' % regressor)

    from sklearn.preprocessing import Imputer
    from sklearn.preprocessing import StandardScaler as Scaler
    from sklearn.pipeline import Pipeline

    # Create and train the learning algorithm
    if config['verbose']:
        print('build_estimator: here are the parameters passed to the learning algorithm...')
        print(sklearn_kwargs)

    # Create the pipeline list
    pipeline = [("imputer", Imputer(missing_values=np.nan, strategy="mean", axis=0))]
    if config['Model']['predict_timeseries']:
        pipeline_timeseries = [("imputer", Imputer(missing_values=np.nan, strategy="mean", axis=0))]

    if not (regressor.startswith('ensemble')):
        # Need to add feature scaling
        pipeline.append(("scaler", Scaler()))
        if config['Model']['predict_timeseries']:
            pipeline_timeseries.append(("scaler", Scaler()))

    # Create the regressor object
    regressor_obj = Regressor(**sklearn_kwargs)
    if ada_boost is not None:
        if config['verbose']:
            print('build_estimator: using Ada boosting...')
        from sklearn.ensemble import AdaBoostRegressor
        regressor_obj = AdaBoostRegressor(regressor_obj, **ada_boost)
    if train_individual:
        if config['verbose']:
            print('build_estimator: training separate models for each parameter...')
        from sklearn.multioutput import MultiOutputRegressor
        multi_regressor = MultiOutputRegressor(regressor_obj, 4)
        pipeline.append(("regressor", multi_regressor))
    else:
        pipeline.append(("regressor", regressor_obj))
    if config['Model']['predict_timeseries']:
        pipeline_timeseries.append(("regressor", regressor_obj))

    # Make the final estimator with a Pipeline
    if config['Model']['predict_timeseries']:
        estimator = TimeSeriesEstimator(Pipeline(pipeline), Pipeline(pipeline_timeseries))
    else:
        estimator = Pipeline(pipeline)

    if rain_tuning is not None and regressor.startswith('ensemble'):
        if config['verbose']:
            print('build_estimator: using rain tuning...')
        rain_kwargs = rain_tuning.copy()
        rain_kwargs.pop('use_raw_rain', None)
        estimator = RainTuningEstimator(estimator, **rain_kwargs)

    # Add bootstrapping if requested
    if bootstrap is not None:
        if config['verbose']:
            print('build_estimator: using bootstrapping ensemble...')
        estimator = BootStrapEnsembleEstimator(estimator, **bootstrap)

    return estimator