jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def __init__(self):
        # Read data transformation parameters
        self.mean_imputer_ = joblib.load('./data_transform_options/mean_imputer.pkl')
        self.zero_imputer_ = Imputer(missing_values=np.nan, strategy='constant', fill_value=0)

        # Define data columns
        self.rate_thresh_list_ = [1, 2, 3, 4]
        self.perc_thresh_list_ = [0.1, 0.2, 0.4]
        self.src_columns_ = np.array(pickle.load(open('./data_transform_options/data_columns.pkl', 'rb')))[:-1]
        self.src_columns_ = np.append(self.src_columns_, ['ratePCA1', 'ratePCA2'])
        for rate_thresh in self.rate_thresh_list_:
            self.src_columns_ = np.append(self.src_columns_, \
                                          ['askCummVolPerc{}'.format(rate_thresh),
                                           'bidCummVolPerc{}'.format(rate_thresh), \
                                           'totalCummVolPerc{}'.format(rate_thresh),
                                           'deltaCummVol{}'.format(rate_thresh)])
        for perc_thresh in self.perc_thresh_list_:
            self.src_columns_ = np.append(self.src_columns_, \
                                          ['askRateMax{}'.format(perc_thresh), 'bidRateMin{}'.format(perc_thresh),
                                           'bidAskSpread{}'.format(perc_thresh)])
        self.src_columns_ = np.append(self.src_columns_, ['askSizeSlope', 'bidSizeSlope', 'bidAskSlopeDiff'])
        self.src_columns_ = np.append(self.src_columns_, ['spreadAskBid0'])
        self.src_columns_ = np.append(self.src_columns_, ['imbalance{}'.format(i) for i in range(15)])
        self.src_columns_ = np.append(self.src_columns_, ['bidSize1_14', 'askSize1_14'])
        self.src_columns_ = np.append(self.src_columns_, ['sizeFlow{}'.format(i) for i in range(1, 9)])
        self.src_columns_ = np.append(self.src_columns_, ['bidAskVolume{}'.format(i) for i in range(6)])

        self.rates_PCA_ = joblib.load('./data_transform_options/rates_PCA.pkl')
        # Historical data accumulation
        self.hist_data_ = np.empty(shape=(0, len(self.src_columns_)))
        self.max_window_ = int(1e3)

        # Core models
        self.xgb1_ = pickle.load(open("./core_models/xgb1_3x_400.pkl", "rb"))
        self.xgb1_columns_ = pickle.load(open("./core_models/xgb1_columns.pkl", "rb"))
        super().__init__()
//  [python]
def __init__(self):
        # Read data transformation parameters
        self.mean_imputer_ = joblib.load('./data_transform_options/mean_imputer.pkl')
        self.zero_imputer_ = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=0)

        # Define data columns
        self.rate_thresh_list_ = [1, 2, 3, 4]
        self.perc_thresh_list_ = [0.1, 0.2, 0.4]
        self.src_columns_ = np.array(pickle.load(open('./data_transform_options/data_columns.pkl', 'rb')))[:-1]
        self.src_columns_ = np.append(self.src_columns_, ['ratePCA1', 'ratePCA2'])
        for rate_thresh in self.rate_thresh_list_:
            self.src_columns_ = np.append(self.src_columns_, \
                                          ['askCummVolPerc{}'.format(rate_thresh),
                                           'bidCummVolPerc{}'.format(rate_thresh), \
                                           'totalCummVolPerc{}'.format(rate_thresh),
                                           'deltaCummVol{}'.format(rate_thresh)])
        for perc_thresh in self.perc_thresh_list_:
            self.src_columns_ = np.append(self.src_columns_, \
                                          ['askRateMax{}'.format(perc_thresh), 'bidRateMin{}'.format(perc_thresh),
                                           'bidAskSpread{}'.format(perc_thresh)])
        self.src_columns_ = np.append(self.src_columns_, ['askSizeSlope', 'bidSizeSlope', 'bidAskSlopeDiff'])
        self.src_columns_ = np.append(self.src_columns_, ['spreadAskBid0'])
        self.src_columns_ = np.append(self.src_columns_, ['imbalance{}'.format(i) for i in range(15)])
        self.src_columns_ = np.append(self.src_columns_, ['bidSize1_14', 'askSize1_14'])
        self.src_columns_ = np.append(self.src_columns_, ['sizeFlow{}'.format(i) for i in range(1, 9)])
        self.src_columns_ = np.append(self.src_columns_, ['bidAskVolume{}'.format(i) for i in range(6)])

        self.rates_PCA_ = joblib.load('./data_transform_options/rates_PCA.pkl')
        # Historical data accumulation
        self.hist_data_ = np.empty(shape=(0, len(self.src_columns_)))
        self.max_window_ = int(1e3)

        # Core models
        self.xgb1_ = pickle.load(open("./core_models/xgb1_3x_400.pkl", "rb"))
        self.xgb1_columns_ = pickle.load(open("./core_models/xgb1_columns.pkl", "rb"))
        super().__init__()
//  [python]
def test_data_reconstruction_drift_calculator_given_wrong_cont_imputer_object_raises_typeerror(  # noqa: D103
    sample_drift_data_with_nans,
):
    with pytest.raises(TypeError):
        DataReconstructionDriftCalculator(
            column_names=['f1', 'f2', 'f3', 'f4'],
            timestamp_column_name='timestamp',
            chunk_period='W',
            imputer_categorical=Imputer(missing_values='NaN', strategy='most_frequent'),
            imputer_continuous=5,
        )
//  [python]
def test_data_reconstruction_drift_calculator_given_wrong_cont_imputer_object_raises_typeerror(  # noqa: D103
    sample_drift_data_with_nans,
):
    with pytest.raises(TypeError):
        DataReconstructionDriftCalculator(
            column_names=['f1', 'f2', 'f3', 'f4'],
            timestamp_column_name='timestamp',
            chunk_period='W',
            imputer_categorical=SimpleImputer(missing_values=np.nan, strategy='most_frequent'),
            imputer_continuous=5,
        )
//  [python]
def test_data_reconstruction_drift_calculator_given_wrong_cat_imputer_strategy_raises_valueerror(  # noqa: D103
    sample_drift_data_with_nans,
):
    with pytest.raises(ValueError):
        DataReconstructionDriftCalculator(
            column_names=['f1', 'f2', 'f3', 'f4'],
            timestamp_column_name='timestamp',
            chunk_period='W',
            imputer_categorical=Imputer(missing_values=np.nan, strategy='median'),
            imputer_continuous=SimpleImputer(missing_values=np.nan, strategy='mean'),
        )
//  [python]
def test_data_reconstruction_drift_calculator_given_wrong_cat_imputer_strategy_raises_valueerror(  # noqa: D103
    sample_drift_data_with_nans,
):
    with pytest.raises(ValueError):
        DataReconstructionDriftCalculator(
            column_names=['f1', 'f2', 'f3', 'f4'],
            timestamp_column_name='timestamp',
            chunk_period='W',
            imputer_categorical=SimpleImputer(missing_values=np.nan, strategy='median'),
            imputer_continuous=SimpleImputer(missing_values=np.nan, strategy='mean'),
        )
//  [python]
def test_simple_imputation_inverse_transform(missing_value):
    # Test inverse_transform feature for np.nan
    X_1 = np.array([
        [9, missing_value, 3, -1],
        [4, -1, 5, 4],
        [6, 7, missing_value, -1],
        [8, 9, 0, missing_value]
    ])

    X_2 = np.array([
        [5, 4, 2, 1],
        [2, 1, missing_value, 3],
        [9, missing_value, 7, 1],
        [6, 4, 2, missing_value]
    ])

    X_3 = np.array([
        [1, missing_value, 5, 9],
        [missing_value, 4, missing_value, missing_value],
        [2, missing_value, 7, missing_value],
        [missing_value, 3, missing_value, 8]
    ])

    X_4 = np.array([
        [1, 1, 1, 3],
        [missing_value, 2, missing_value, 1],
        [2, 3, 3, 4],
        [missing_value, 4, missing_value, 2]
    ])

    imputer = Imputer(missing_values=missing_value, strategy='mean',
                            add_indicator=True)

    X_1_trans = imputer.fit_transform(X_1)
    X_1_inv_trans = imputer.inverse_transform(X_1_trans)

    X_2_trans = imputer.transform(X_2)  # test on new data
    X_2_inv_trans = imputer.inverse_transform(X_2_trans)

    assert_array_equal(X_1_inv_trans, X_1)
    assert_array_equal(X_2_inv_trans, X_2)

    for X in [X_3, X_4]:
        X_trans = imputer.fit_transform(X)
        X_inv_trans = imputer.inverse_transform(X_trans)
        assert_array_equal(X_inv_trans, X)
//  [python]
def test_simple_imputation_inverse_transform(missing_value):
    # Test inverse_transform feature for np.nan
    X_1 = np.array([
        [9, missing_value, 3, -1],
        [4, -1, 5, 4],
        [6, 7, missing_value, -1],
        [8, 9, 0, missing_value]
    ])

    X_2 = np.array([
        [5, 4, 2, 1],
        [2, 1, missing_value, 3],
        [9, missing_value, 7, 1],
        [6, 4, 2, missing_value]
    ])

    X_3 = np.array([
        [1, missing_value, 5, 9],
        [missing_value, 4, missing_value, missing_value],
        [2, missing_value, 7, missing_value],
        [missing_value, 3, missing_value, 8]
    ])

    X_4 = np.array([
        [1, 1, 1, 3],
        [missing_value, 2, missing_value, 1],
        [2, 3, 3, 4],
        [missing_value, 4, missing_value, 2]
    ])

    imputer = SimpleImputer(missing_values=missing_value, strategy='mean',
                            add_indicator=True)

    X_1_trans = imputer.fit_transform(X_1)
    X_1_inv_trans = imputer.inverse_transform(X_1_trans)

    X_2_trans = imputer.transform(X_2)  # test on new data
    X_2_inv_trans = imputer.inverse_transform(X_2_trans)

    assert_array_equal(X_1_inv_trans, X_1)
    assert_array_equal(X_2_inv_trans, X_2)

    for X in [X_3, X_4]:
        X_trans = imputer.fit_transform(X)
        X_inv_trans = imputer.inverse_transform(X_trans)
        assert_array_equal(X_inv_trans, X)
//  [python]
def general_preprocessor(*, number_scaler=None):
    cat_steps = [('imputer_cat', Imputer(strategy='constant', fill_value='')),
                 ('encoder', SafeOrdinalEncoder())]
    num_steps = [('imputer_num', Imputer(strategy='mean')),
                 # ('scaler', StandardScaler()),
                 ]
    if number_scaler is True:
        num_steps.append(('scaler', StandardScaler()))

    cat_transformer = Pipeline(steps=cat_steps)
    num_transformer = Pipeline(steps=num_steps)
    bool_transformer = SafeOrdinalEncoder()

    cat_selector = make_column_selector(dtype_include=['object', 'category'])
    bool_selector = make_column_selector(dtype_include='bool')
    num_selector = make_column_selector(dtype_include='number', dtype_exclude='timedelta')

    preprocessor = DataFrameMapper(
        features=[(cat_selector, cat_transformer),
                  (bool_selector, bool_transformer),
                  (num_selector, num_transformer)],
        input_df=True,
        df_out=True)
    return preprocessor
//  [python]
def general_preprocessor(*, number_scaler=None):
    cat_steps = [('imputer_cat', SimpleImputer(strategy='constant', fill_value='')),
                 ('encoder', SafeOrdinalEncoder())]
    num_steps = [('imputer_num', SimpleImputer(strategy='mean')),
                 # ('scaler', StandardScaler()),
                 ]
    if number_scaler is True:
        num_steps.append(('scaler', StandardScaler()))

    cat_transformer = Pipeline(steps=cat_steps)
    num_transformer = Pipeline(steps=num_steps)
    bool_transformer = SafeOrdinalEncoder()

    cat_selector = make_column_selector(dtype_include=['object', 'category'])
    bool_selector = make_column_selector(dtype_include='bool')
    num_selector = make_column_selector(dtype_include='number', dtype_exclude='timedelta')

    preprocessor = DataFrameMapper(
        features=[(cat_selector, cat_transformer),
                  (bool_selector, bool_transformer),
                  (num_selector, num_transformer)],
        input_df=True,
        df_out=True)
    return preprocessor
//  [python]
def make_regressor_pipeline(X: pd.DataFrame) -> Pipeline:
    """
    Make the regressor pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X
        X containing all the required features for training

    Returns
    -------
    Pipeline
        Regressor pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", Imputer(strategy="mean")), ("standardize", StandardScaler())]
    )
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = create_regression_model()

    # pipeline with preprocessor and estimator bundled
    regressor_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return regressor_pipeline
//  [python]
def make_regressor_pipeline(X: pd.DataFrame) -> Pipeline:
    """
    Make the regressor pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X
        X containing all the required features for training

    Returns
    -------
    Pipeline
        Regressor pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", SimpleImputer(strategy="mean")), ("standardize", StandardScaler())]
    )
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = create_regression_model()

    # pipeline with preprocessor and estimator bundled
    regressor_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return regressor_pipeline
//  [python]
def scale_and_impute(self, train, test, features, extra_feats):
        trans_train = train[features].values
        trans_test = test[features].values

        # Scale data if linear
        my_scaler = sklearn.preprocessing.Normalizer(
            norm='l1',
            copy=False)
        my_scaler.fit_transform(trans_train)
        my_scaler.transform(trans_test)

        # Impute data
        imp_mean = sklearn.preprocessing.Imputer(
            #missing_values=MISSING,
            copy=False,
            strategy='mean')
        imp_mean.fit_transform(trans_train)
        imp_mean.transform(trans_test)

        # recreate dataframe
        trans_train = np.append(trans_train, train[extra_feats].values, axis=1)
        df_trans_train = pd.DataFrame(trans_train, columns=features+extra_feats)
        train = df_trans_train

        trans_test = np.append(trans_test, test[extra_feats].values, axis=1)
        df_trans_test = pd.DataFrame(trans_test, columns=features+extra_feats)
        test = df_trans_test

        print("Value distribution of 'rdb0.glue' in test:\n%s" % test["rdb0.glue"].value_counts())
        print("Value distribution of 'rdb0.glue' in train:\n%s" % train["rdb0.glue"].value_counts())

        return train, test
//  [python]
def scale_and_impute(self, train, test, features, extra_feats):
        trans_train = train[features].values
        trans_test = test[features].values

        # Scale data if linear
        my_scaler = sklearn.preprocessing.Normalizer(
            norm='l1',
            copy=False)
        my_scaler.fit_transform(trans_train)
        my_scaler.transform(trans_test)

        # Impute data
        imp_mean = sklearn.impute.SimpleImputer(
            #missing_values=MISSING,
            copy=False,
            strategy='mean')
        imp_mean.fit_transform(trans_train)
        imp_mean.transform(trans_test)

        # recreate dataframe
        trans_train = np.append(trans_train, train[extra_feats].values, axis=1)
        df_trans_train = pd.DataFrame(trans_train, columns=features+extra_feats)
        train = df_trans_train

        trans_test = np.append(trans_test, test[extra_feats].values, axis=1)
        df_trans_test = pd.DataFrame(trans_test, columns=features+extra_feats)
        test = df_trans_test

        print("Value distribution of 'rdb0.glue' in test:\n%s" % test["rdb0.glue"].value_counts())
        print("Value distribution of 'rdb0.glue' in train:\n%s" % train["rdb0.glue"].value_counts())

        return train, test
//  [python]
def get_column_transformer() -> ColumnTransformer:
    num_pipeline = Pipeline(
        steps=[("impute", Imputer(strategy="mean")), ("scale", MinMaxScaler())]
    )
    cat_pipeline = Pipeline(
        steps=[
            ("impute", Imputer(strategy="most_frequent")),
            ("one-hot", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    col_trans = ColumnTransformer(
        transformers=[
            (
                "num_pipeline",
                num_pipeline,
                make_column_selector(dtype_include=np.number),
            ),
            ("cat_pipeline", cat_pipeline, make_column_selector(dtype_include=object)),
        ],
        remainder="drop",
        n_jobs=-1,
    )
    return col_trans
//  [python]
def get_column_transformer() -> ColumnTransformer:
    num_pipeline = Pipeline(
        steps=[("impute", SimpleImputer(strategy="mean")), ("scale", MinMaxScaler())]
    )
    cat_pipeline = Pipeline(
        steps=[
            ("impute", SimpleImputer(strategy="most_frequent")),
            ("one-hot", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    col_trans = ColumnTransformer(
        transformers=[
            (
                "num_pipeline",
                num_pipeline,
                make_column_selector(dtype_include=np.number),
            ),
            ("cat_pipeline", cat_pipeline, make_column_selector(dtype_include=object)),
        ],
        remainder="drop",
        n_jobs=-1,
    )
    return col_trans
//  [python]
def testMultiTaskREG(self):
        self.settings.target_props = [
            [
                TargetProperty.fromDict(
                    {
                        "name": "CL",
                        "task": TargetTasks.REGRESSION,
                        "imputer": Imputer(strategy="mean"),
                    }
                ),
                TargetProperty.fromDict(
                    {
                        "name": "fu",
                        "task": TargetTasks.REGRESSION,
                        "imputer": Imputer(strategy="mean"),
                    }
                ),
            ]
        ]
        self.settings.models = [
            SklearnModel(
                name="RandomForestRegressor",
                alg=RandomForestRegressor,
                base_dir=f"{self.generatedPath}/models",
            ),
            SklearnModel(
                name="KNeighborsRegressor",
                alg=KNeighborsRegressor,
                base_dir=f"{self.generatedPath}/models",
            ),
        ]
        self.settings.assessors = [
            CrossValAssessor(
                scoring="r2",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            CrossValAssessor(
                scoring="neg_mean_squared_error",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            TestSetAssessor(scoring="r2", split_multitask_scores=True),
            TestSetAssessor(
                scoring="neg_mean_squared_error", split_multitask_scores=True
            ),
        ]
        self.checkSettings()
        results = self.benchmark.run(raise_errors=True)
        self.checkRunResults(results)
        self.checkSettings()
//  [python]
def testMultiTaskREG(self):
        self.settings.target_props = [
            [
                TargetProperty.fromDict(
                    {
                        "name": "CL",
                        "task": TargetTasks.REGRESSION,
                        "imputer": SimpleImputer(strategy="mean"),
                    }
                ),
                TargetProperty.fromDict(
                    {
                        "name": "fu",
                        "task": TargetTasks.REGRESSION,
                        "imputer": SimpleImputer(strategy="mean"),
                    }
                ),
            ]
        ]
        self.settings.models = [
            SklearnModel(
                name="RandomForestRegressor",
                alg=RandomForestRegressor,
                base_dir=f"{self.generatedPath}/models",
            ),
            SklearnModel(
                name="KNeighborsRegressor",
                alg=KNeighborsRegressor,
                base_dir=f"{self.generatedPath}/models",
            ),
        ]
        self.settings.assessors = [
            CrossValAssessor(
                scoring="r2",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            CrossValAssessor(
                scoring="neg_mean_squared_error",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            TestSetAssessor(scoring="r2", split_multitask_scores=True),
            TestSetAssessor(
                scoring="neg_mean_squared_error", split_multitask_scores=True
            ),
        ]
        self.checkSettings()
        results = self.benchmark.run(raise_errors=True)
        self.checkRunResults(results)
        self.checkSettings()
//  [python]
def __init__(
        self,
        missing_values=np.nan,
        strategy="mean",
        fill_value=None,
        copy=True,
        add_indicator=False,
        keep_empty_features=False,
        sketch_name="KLL",
        k=200,
        is_hra=True,
        FL_type=None,
        role=None,
        channel=None,
    ):
        super().__init__(FL_type, role, channel)
        if self.FL_type == "H" and strategy != "constant":
            self.check_channel()
        self.sketch_name = sketch_name
        self.k = k
        self.is_hra = is_hra
        self.module = Imputer(
            missing_values=missing_values,
            strategy=strategy,
            fill_value=fill_value,
            copy=copy,
            add_indicator=add_indicator,
            keep_empty_features=keep_empty_features,
        )
        if FL_type == "H":
            self.missing_values = missing_values
            self.strategy = strategy
            self.copy = copy
            self.add_indicator = add_indicator
//  [python]
def __init__(
        self,
        missing_values=np.nan,
        strategy="mean",
        fill_value=None,
        copy=True,
        add_indicator=False,
        keep_empty_features=False,
        sketch_name="KLL",
        k=200,
        is_hra=True,
        FL_type=None,
        role=None,
        channel=None,
    ):
        super().__init__(FL_type, role, channel)
        if self.FL_type == "H" and strategy != "constant":
            self.check_channel()
        self.sketch_name = sketch_name
        self.k = k
        self.is_hra = is_hra
        self.module = SKL_SimpleImputer(
            missing_values=missing_values,
            strategy=strategy,
            fill_value=fill_value,
            copy=copy,
            add_indicator=add_indicator,
            keep_empty_features=keep_empty_features,
        )
        if FL_type == "H":
            self.missing_values = missing_values
            self.strategy = strategy
            self.copy = copy
            self.add_indicator = add_indicator
//  [python]
def testMultiTaskCLS(self):
        """Run the test benchmark."""
        self.settings.target_props = [
            [
                TargetProperty.fromDict(
                    {
                        "name": "CL",
                        "task": TargetTasks.SINGLECLASS,
                        "th": [10],
                        "imputer": Imputer(strategy="mean"),
                    }
                ),
                TargetProperty.fromDict(
                    {
                        "name": "fu",
                        "task": TargetTasks.SINGLECLASS,
                        "th": [0.3],
                        "imputer": Imputer(strategy="mean"),
                    }
                ),
            ]
        ]
        self.settings.models = [
            SklearnModel(
                name="RandomForestClassifier",
                alg=RandomForestClassifier,
                base_dir=f"{self.generatedPath}/models",
            ),
            SklearnModel(
                name="KNeighborsClassifier",
                alg=KNeighborsClassifier,
                base_dir=f"{self.generatedPath}/models",
            ),
        ]
        self.settings.assessors = [
            CrossValAssessor(
                scoring="roc_auc",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            CrossValAssessor(
                scoring="matthews_corrcoef",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                use_proba=False,
                split_multitask_scores=True,
            ),
            TestSetAssessor(scoring="roc_auc", split_multitask_scores=True),
            TestSetAssessor(
                scoring="matthews_corrcoef",
                use_proba=False,
                split_multitask_scores=True,
            ),
        ]
        self.checkSettings()
        results = self.benchmark.run(raise_errors=True)
        self.checkRunResults(results)
        self.checkSettings()
//  [python]
def testMultiTaskCLS(self):
        """Run the test benchmark."""
        self.settings.target_props = [
            [
                TargetProperty.fromDict(
                    {
                        "name": "CL",
                        "task": TargetTasks.SINGLECLASS,
                        "th": [10],
                        "imputer": SimpleImputer(strategy="mean"),
                    }
                ),
                TargetProperty.fromDict(
                    {
                        "name": "fu",
                        "task": TargetTasks.SINGLECLASS,
                        "th": [0.3],
                        "imputer": SimpleImputer(strategy="mean"),
                    }
                ),
            ]
        ]
        self.settings.models = [
            SklearnModel(
                name="RandomForestClassifier",
                alg=RandomForestClassifier,
                base_dir=f"{self.generatedPath}/models",
            ),
            SklearnModel(
                name="KNeighborsClassifier",
                alg=KNeighborsClassifier,
                base_dir=f"{self.generatedPath}/models",
            ),
        ]
        self.settings.assessors = [
            CrossValAssessor(
                scoring="roc_auc",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                split_multitask_scores=True,
            ),
            CrossValAssessor(
                scoring="matthews_corrcoef",
                split=KFold(n_splits=self.nFolds, shuffle=True, random_state=self.seed),
                use_proba=False,
                split_multitask_scores=True,
            ),
            TestSetAssessor(scoring="roc_auc", split_multitask_scores=True),
            TestSetAssessor(
                scoring="matthews_corrcoef",
                use_proba=False,
                split_multitask_scores=True,
            ),
        ]
        self.checkSettings()
        results = self.benchmark.run(raise_errors=True)
        self.checkRunResults(results)
        self.checkSettings()
//  [python]
def buildmodel():
    print("buildmodel() is called.")

    # Get Datasets
    trainds = Dataset.get_by_name(current_ws,trainDatasetName)
    testds = Dataset.get_by_name(current_ws,testDatasetName)
    traindf = trainds.to_pandas_dataframe()
    testdf = testds.to_pandas_dataframe()

    print("Training rows => ",traindf.shape[0])
    print("Test rows => ",testdf.shape[0])

    # Train / Test datasets
    X_train = traindf[features]
    y_train = traindf[targetColumnName]

    X_test = testdf[features]
    y_test = testdf[targetColumnName]

    # Replace null values with "median" and normalize all numeric values using MinMaxScaler
    numeric_transformer = Pipeline(steps=[('imputer', Imputer(strategy='median')), ('scaler', MinMaxScaler())])
    transformations = ColumnTransformer(transformers=[('num', numeric_transformer, X_train.columns)])

    # Build classifier pipeline with preprocessing steps as above, and LightGBM model
    # Only use training dataset for build the model
    classifierPipeline = Pipeline(steps=[('preprocessor', transformations),('classifier', LGBMClassifier())])
    model = classifierPipeline.fit(X_train, y_train)

    # Run the model and extract predictions
    y_pred = classifierPipeline.predict(X_test)

    # Score the model against true values
    trainingScore = classifierPipeline.score(X_train, y_train)
    testScore = classifierPipeline.score(X_test, y_test)
    print('Training set score: {:.4f}'.format(trainingScore))
    print('Test set score: {:.4f}'.format(testScore))
    print(classification_report(y_test, y_pred))

    current_run.log("training_accuracy",trainingScore)
    current_run.log("test_accuracy",testScore)

    # Save and Register Model
    pickle.dump(classifierPipeline, open(modelFileName, 'wb'))
    modeltags = { 
        "experiment": current_run.experiment.name,
        "run_id": current_run.id,
        "train_dataset_name" : trainds.name,
        "train_dataset_version" : trainds.version,
        "test_dataset_name" : testds.name,
        "test_dataset_version" : testds.version
    }
    model = Model.register(model_path=modelFileName, model_name=modelName, 
                           tags=modeltags, description="Light GBM model model for iiot quality prediction",workspace=current_ws)

    print("model built and registered")
//  [python]
def buildmodel():
    print("buildmodel() is called.")

    # Get Datasets
    trainds = Dataset.get_by_name(current_ws,trainDatasetName)
    testds = Dataset.get_by_name(current_ws,testDatasetName)
    traindf = trainds.to_pandas_dataframe()
    testdf = testds.to_pandas_dataframe()

    print("Training rows => ",traindf.shape[0])
    print("Test rows => ",testdf.shape[0])

    # Train / Test datasets
    X_train = traindf[features]
    y_train = traindf[targetColumnName]

    X_test = testdf[features]
    y_test = testdf[targetColumnName]

    # Replace null values with "median" and normalize all numeric values using MinMaxScaler
    numeric_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='median')), ('scaler', MinMaxScaler())])
    transformations = ColumnTransformer(transformers=[('num', numeric_transformer, X_train.columns)])

    # Build classifier pipeline with preprocessing steps as above, and LightGBM model
    # Only use training dataset for build the model
    classifierPipeline = Pipeline(steps=[('preprocessor', transformations),('classifier', LGBMClassifier())])
    model = classifierPipeline.fit(X_train, y_train)

    # Run the model and extract predictions
    y_pred = classifierPipeline.predict(X_test)

    # Score the model against true values
    trainingScore = classifierPipeline.score(X_train, y_train)
    testScore = classifierPipeline.score(X_test, y_test)
    print('Training set score: {:.4f}'.format(trainingScore))
    print('Test set score: {:.4f}'.format(testScore))
    print(classification_report(y_test, y_pred))

    current_run.log("training_accuracy",trainingScore)
    current_run.log("test_accuracy",testScore)

    # Save and Register Model
    pickle.dump(classifierPipeline, open(modelFileName, 'wb'))
    modeltags = { 
        "experiment": current_run.experiment.name,
        "run_id": current_run.id,
        "train_dataset_name" : trainds.name,
        "train_dataset_version" : trainds.version,
        "test_dataset_name" : testds.name,
        "test_dataset_version" : testds.version
    }
    model = Model.register(model_path=modelFileName, model_name=modelName, 
                           tags=modeltags, description="Light GBM model model for iiot quality prediction",workspace=current_ws)

    print("model built and registered")
//  [python]
def process_data(self, data):
        imputer = Imputer(missing_values='NaN', strategy='mean')
        return pd.DataFrame(imputer.fit_transform(pd.DataFrame(data)))
//  [python]
def process_data(self, data):
        imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
        return pd.DataFrame(imputer.fit_transform(pd.DataFrame(data)))
//  [python]
def _fit_predict(cls, X_train, y_train, X_test):
        """Fit a pipeline to the training data and then use it to make prediction on test data."""
        model = Pipeline([
            ('imputer', Imputer()),
            ('scalar', RobustScaler()),
            ('classifier', cls._get_classifier()),
        ])
        model.fit(X_train, y_train)

        return model.predict_proba(X_test)[:, 1]
//  [python]
def _fit_predict(cls, X_train, y_train, X_test):
        """Fit a pipeline to the training data and then use it to make prediction on test data."""
        model = Pipeline([
            ('imputer', SimpleImputer()),
            ('scalar', RobustScaler()),
            ('classifier', cls._get_classifier()),
        ])
        model.fit(X_train, y_train)

        return model.predict_proba(X_test)[:, 1]
//  [python]
idx = 30000
X_train, Y_train = data[:idx, :], targets[:idx]
X_test, Y_test = data[idx + 1 :, :], targets[idx + 1 :]

# feature transformation pipeline
ordinal_features = [
    x for x in range(len(feature_names)) if x not in list(category_map.keys())
]
ordinal_transformer = Pipeline(
    steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
idx = 30000
X_train, Y_train = data[:idx, :], targets[:idx]
X_test, Y_test = data[idx + 1 :, :], targets[idx + 1 :]

# feature transformation pipeline
ordinal_features = [
    x for x in range(len(feature_names)) if x not in list(category_map.keys())
]
ordinal_transformer = Pipeline(
    steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
def create_complex_classification_pipeline(
        X_train, y_train, continuous_features, categorical_features):
    """Create a complex sklearn pipeline for classification.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    param continuous_features: The continuous features.
    type continuous_features: list
    param categorical_features: The categorical features.
    type categorical_features: list
    return: A complex sklearn pipeline for classification.
    rtype: sklearn.pipeline.Pipeline
    """
    # We create the preprocessing pipelines for both
    # numeric and categorical data.
    numeric_transformer = Pipeline(steps=[
        ("imputer", Imputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestClassifier())])
    return pipeline.fit(X_train, y_train)
//  [python]
def create_complex_classification_pipeline(
        X_train, y_train, continuous_features, categorical_features):
    """Create a complex sklearn pipeline for classification.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    param continuous_features: The continuous features.
    type continuous_features: list
    param categorical_features: The categorical features.
    type categorical_features: list
    return: A complex sklearn pipeline for classification.
    rtype: sklearn.pipeline.Pipeline
    """
    # We create the preprocessing pipelines for both
    # numeric and categorical data.
    numeric_transformer = Pipeline(steps=[
        ("imputer", SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestClassifier())])
    return pipeline.fit(X_train, y_train)
//  [python]
def create_complex_regression_pipeline(
        X_train, y_train, continuous_features, categorical_features):
    """Create a complex sklearn pipeline for regression.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    param continuous_features: The continuous features.
    type continuous_features: list
    param categorical_features: The categorical features.
    type categorical_features: list
    return: A complex sklearn pipeline for regression.
    rtype: sklearn.pipeline.Pipeline
    """
    # We create the preprocessing pipelines for both
    # numeric and categorical data.
    numeric_transformer = Pipeline(steps=[
        ("imputer", Imputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestRegressor())])
    return pipeline.fit(X_train, y_train)
//  [python]
def create_complex_regression_pipeline(
        X_train, y_train, continuous_features, categorical_features):
    """Create a complex sklearn pipeline for regression.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    param continuous_features: The continuous features.
    type continuous_features: list
    param categorical_features: The categorical features.
    type categorical_features: list
    return: A complex sklearn pipeline for regression.
    rtype: sklearn.pipeline.Pipeline
    """
    # We create the preprocessing pipelines for both
    # numeric and categorical data.
    numeric_transformer = Pipeline(steps=[
        ("imputer", SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestRegressor())])
    return pipeline.fit(X_train, y_train)
//  [python]
def create_titanic_pipeline(X_train, y_train):
    """Create a sklearn pipeline for the titanic dataset.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    return: A sklearn pipeline for the titanic dataset.
    rtype: sklearn.pipeline.Pipeline
    """
    def conv(X):
        """Convert a pandas series to a numpy array.

        param X: The data.
        type X: pandas.Series
        return: The data as a numpy array.
        rtype: numpy.ndarray
        """
        if isinstance(X, pd.Series):
            return X.values
        return X

    many_to_one_transformer = \
        FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
    many_to_many_transformer = \
        FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1),
                conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
    # for older scikit-learn versions use sparse, for newer sparse_output:
    if version.parse(sklearn.__version__) < version.parse('1.2'):
        ohe_params = {"sparse": False}
    else:
        ohe_params = {"sparse_output": False}
    transformations = ColumnTransformer([
        ("age_fare_1", Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())
        ]), ["age", "fare"]),
        ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
        ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
        ("embarked", Pipeline(steps=[
            ("imputer",
                Imputer(strategy='constant', fill_value='missing')),
            ("encoder", OneHotEncoder(**ohe_params))]), ["embarked"]),
        ("sex_pclass", OneHotEncoder(**ohe_params), ["sex", "pclass"])
    ])
    clf = Pipeline(steps=[('preprocessor', transformations),
                          ('classifier',
                           LogisticRegression(solver='lbfgs'))])
    clf.fit(X_train, y_train)
    return clf
//  [python]
def create_titanic_pipeline(X_train, y_train):
    """Create a sklearn pipeline for the titanic dataset.

    param X_train: The training data.
    type X_train: numpy.ndarray or pandas.DataFrame
    param y_train: The training labels.
    type y_train: numpy.ndarray or pandas.DataFrame
    return: A sklearn pipeline for the titanic dataset.
    rtype: sklearn.pipeline.Pipeline
    """
    def conv(X):
        """Convert a pandas series to a numpy array.

        param X: The data.
        type X: pandas.Series
        return: The data as a numpy array.
        rtype: numpy.ndarray
        """
        if isinstance(X, pd.Series):
            return X.values
        return X

    many_to_one_transformer = \
        FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
    many_to_many_transformer = \
        FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1),
                conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
    # for older scikit-learn versions use sparse, for newer sparse_output:
    if version.parse(sklearn.__version__) < version.parse('1.2'):
        ohe_params = {"sparse": False}
    else:
        ohe_params = {"sparse_output": False}
    transformations = ColumnTransformer([
        ("age_fare_1", Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
        ]), ["age", "fare"]),
        ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
        ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
        ("embarked", Pipeline(steps=[
            ("imputer",
                SimpleImputer(strategy='constant', fill_value='missing')),
            ("encoder", OneHotEncoder(**ohe_params))]), ["embarked"]),
        ("sex_pclass", OneHotEncoder(**ohe_params), ["sex", "pclass"])
    ])
    clf = Pipeline(steps=[('preprocessor', transformations),
                          ('classifier',
                           LogisticRegression(solver='lbfgs'))])
    clf.fit(X_train, y_train)
    return clf
//  [python]
def create_pipelines(self, cat_cols, cont_cols):
        self.model_pipelines = []
        for scaler in self.scalers:
            pipeline_num = Pipeline(steps=[('imputer', Imputer(strategy='median')),
                                           ('scaling',scaler)])
            for encoder in self.encoders:
                pipeline_cat = Pipeline(steps=[('imputer', Imputer(strategy='constant', fill_value='missing')),
                                               ('encoder',encoder)])
                preprocessor = make_column_transformer((pipeline_num, cont_cols),(pipeline_cat, cat_cols))
                
                for estimator in self.estimators:
                    pipeline  = make_pipeline(preprocessor, estimator)
                    self.model_pipelines.append(pipeline)
//  [python]
def create_pipelines(self, cat_cols, cont_cols):
        self.model_pipelines = []
        for scaler in self.scalers:
            pipeline_num = Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),
                                           ('scaling',scaler)])
            for encoder in self.encoders:
                pipeline_cat = Pipeline(steps=[('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
                                               ('encoder',encoder)])
                preprocessor = make_column_transformer((pipeline_num, cont_cols),(pipeline_cat, cat_cols))
                
                for estimator in self.estimators:
                    pipeline  = make_pipeline(preprocessor, estimator)
                    self.model_pipelines.append(pipeline)
//  [python]
def __init__(self, *,
                 hyperparams: Hyperparams,
                 random_seed: int = 0,
                 docker_containers: Dict[str, DockerContainer] = None,
                 _verbose: int = 0) -> None:

        super().__init__(hyperparams=hyperparams, random_seed=random_seed, docker_containers=docker_containers)
        
        # False
        self._clf = Imputer(
              missing_values=self.hyperparams['missing_values'],
              strategy=self.hyperparams['strategy'],
              axis=0,
              copy=True,
              fill_value=self.hyperparams['fill_value'],
              verbose=_verbose
        )
        
        self._inputs = None
        self._outputs = None
        self._training_inputs = None
        self._training_outputs = None
        self._target_names = None
        self._training_indices = None
        self._target_column_indices = None
        self._target_columns_metadata: List[OrderedDict] = None
        self._input_column_names = None
        self._fitted = False
//  [python]
def __init__(self, *,
                 hyperparams: Hyperparams,
                 random_seed: int = 0,
                 docker_containers: Dict[str, DockerContainer] = None,
                 _verbose: int = 0) -> None:

        super().__init__(hyperparams=hyperparams, random_seed=random_seed, docker_containers=docker_containers)
        
        # False
        self._clf = SimpleImputer(
              missing_values=self.hyperparams['missing_values'],
              strategy=self.hyperparams['strategy'],
              add_indicator=self.hyperparams['add_indicator'],
              fill_value=self.hyperparams['fill_value'],
              verbose=_verbose
        )
        
        self._inputs = None
        self._outputs = None
        self._training_inputs = None
        self._training_outputs = None
        self._target_names = None
        self._training_indices = None
        self._target_column_indices = None
        self._target_columns_metadata: List[OrderedDict] = None
        self._input_column_names = None
        self._fitted = False