jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_pipeline_compatibility(strategy: str) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    args = STRATEGIES[strategy][0]
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
        }
    )
    y = np.array(
        [
            [0, 0, 1], [0, 0, 1],
            [1, 1, 0], [1, 0, 1],
            [1, 0, 1], [1, 1, 1]
        ]
    )
    numeric_preprocessor = Pipeline(
        [
            ("imputer", Imputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(
        preprocessor,
        MultiOutputClassifier(LogisticRegression())
    )
    pipe.fit(X, y)
    mapie = MapieMultiLabelClassifier(
        estimator=pipe,
        method=args["method"],
        metric_control=args["metric_control"],
        random_state=random_state
    )

    mapie.fit(X, y)
    mapie.predict(X, bound=args["bound"], delta=.1)
//  [python]
def test_pipeline_compatibility(strategy: str) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    args = STRATEGIES[strategy][0]
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
        }
    )
    y = np.array(
        [
            [0, 0, 1], [0, 0, 1],
            [1, 1, 0], [1, 0, 1],
            [1, 0, 1], [1, 1, 1]
        ]
    )
    numeric_preprocessor = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(
        preprocessor,
        MultiOutputClassifier(LogisticRegression())
    )
    pipe.fit(X, y)
    mapie = MapieMultiLabelClassifier(
        estimator=pipe,
        method=args["method"],
        metric_control=args["metric_control"],
        random_state=random_state
    )

    mapie.fit(X, y)
    mapie.predict(X, bound=args["bound"], delta=.1)
//  [python]
def learn_or_load_imputer(dataset, impute_file_name="feature_imputer.joblib"):
    """
    learn a new feature selector if the given file does not exist
    :param dataset: the features dataset
    :param impute_file_name: the given filename of the serialized feature selector
    :return: the feature selector
    """
    if not os.path.exists(impute_file_name):
        t0 = time.time()
        selector = Imputer(missing_values=0, strategy="most_frequent")
        selector.fit(dataset)
        t1 = time.time()
        print("It takes {} seconds to impute zeros for train set".format(t1 - t0))
        dump(selector, impute_file_name)
    else:
        selector = load(impute_file_name)
    return selector
//  [python]
def learn_or_load_imputer(dataset, impute_file_name="feature_imputer.joblib"):
    """
    learn a new feature selector if the given file does not exist
    :param dataset: the features dataset
    :param impute_file_name: the given filename of the serialized feature selector
    :return: the feature selector
    """
    if not os.path.exists(impute_file_name):
        t0 = time.time()
        selector = SimpleImputer(missing_values=0, strategy="most_frequent")
        selector.fit(dataset)
        t1 = time.time()
        print("It takes {} seconds to impute zeros for train set".format(t1 - t0))
        dump(selector, impute_file_name)
    else:
        selector = load(impute_file_name)
    return selector
//  [python]
numeric_selector = make_column_selector(dtype_include=np.number)

# This selector tells sklearn which columns in a pd.DataFrame are categorical
# Note that it will return True for text columns as well
# This means that text variables will be be treated as both text AND categoricals
# This is ok, but if you don't like it, you could write a more complicated is_categorical function
categorical_selector = make_column_selector(dtype_include=object)


##############################
### Preprocessing
##############################

# For numerics we:
# 1. Impute missing values with the median, adding a missing value indicator
# 2. Then center and scale the data
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)
//  [python]
def get_models(position, horizon):
    if position == "GK":
        model_class = GKModel
    else:
        model_class = PointsModel
    return {
        **{
            f"lasso_{alpha}": model_class(
                model=make_pipeline(
                    Imputer(), StandardScaler(), Lasso(alpha=alpha)
                ),
                horizon=horizon,
            )
            for alpha in [0.001, 0.01, 0.1]
        },
        **{
            "linear_regression": model_class(
                model=make_pipeline(
                    Imputer(), StandardScaler(), LinearRegression()
                ),
                horizon=horizon,
            )
        },
    }
//  [python]
def get_models(position, horizon):
    if position == "GK":
        model_class = GKModel
    else:
        model_class = PointsModel
    return {
        **{
            f"lasso_{alpha}": model_class(
                model=make_pipeline(
                    SimpleImputer(), StandardScaler(), Lasso(alpha=alpha)
                ),
                horizon=horizon,
            )
            for alpha in [0.001, 0.01, 0.1]
        },
        **{
            "linear_regression": model_class(
                model=make_pipeline(
                    SimpleImputer(), StandardScaler(), LinearRegression()
                ),
                horizon=horizon,
            )
        },
    }
//  [python]
def train_12ECG_classifier(input_directory, output_directory):
    # Load data.
    print('Loading data...')

    header_files = []
    for f in os.listdir(input_directory):
        g = os.path.join(input_directory, f)
        if not f.lower().startswith('.') and f.lower().endswith('hea') and os.path.isfile(g):
            header_files.append(g)

    classes = get_classes(input_directory, header_files)
    num_classes = len(classes)
    num_files = len(header_files)
    recordings = list()
    headers = list()

    for i in range(num_files):
        recording, header = load_challenge_data(header_files[i])
        recordings.append(recording)
        headers.append(header)

    # Train model.
    print('Training model...')

    features = list()
    labels = list()

    for i in range(num_files):
        recording = recordings[i]
        header = headers[i]

        tmp = get_12ECG_features(recording, header)
        features.append(tmp)

        for l in header:
            if l.startswith('#Dx:'):
                labels_act = np.zeros(num_classes)
                arrs = l.strip().split(' ')
                for arr in arrs[1].split(','):
                    class_index = classes.index(arr.rstrip()) # Only use first positive index
                    labels_act[class_index] = 1
        labels.append(labels_act)

    features = np.array(features)
    labels = np.array(labels)

    # Replace NaN values with mean values
    imputer=Imputer().fit(features)
    features=imputer.transform(features)

    # Train the classifier
    model = RandomForestClassifier().fit(features,labels)

    # Save model.
    print('Saving model...')

    final_model={'model':model, 'imputer':imputer,'classes':classes}

    filename = os.path.join(output_directory, 'finalized_model.sav')
    joblib.dump(final_model, filename, protocol=0)
//  [python]
def train_12ECG_classifier(input_directory, output_directory):
    # Load data.
    print('Loading data...')

    header_files = []
    for f in os.listdir(input_directory):
        g = os.path.join(input_directory, f)
        if not f.lower().startswith('.') and f.lower().endswith('hea') and os.path.isfile(g):
            header_files.append(g)

    classes = get_classes(input_directory, header_files)
    num_classes = len(classes)
    num_files = len(header_files)
    recordings = list()
    headers = list()

    for i in range(num_files):
        recording, header = load_challenge_data(header_files[i])
        recordings.append(recording)
        headers.append(header)

    # Train model.
    print('Training model...')

    features = list()
    labels = list()

    for i in range(num_files):
        recording = recordings[i]
        header = headers[i]

        tmp = get_12ECG_features(recording, header)
        features.append(tmp)

        for l in header:
            if l.startswith('#Dx:'):
                labels_act = np.zeros(num_classes)
                arrs = l.strip().split(' ')
                for arr in arrs[1].split(','):
                    class_index = classes.index(arr.rstrip()) # Only use first positive index
                    labels_act[class_index] = 1
        labels.append(labels_act)

    features = np.array(features)
    labels = np.array(labels)

    # Replace NaN values with mean values
    imputer=SimpleImputer().fit(features)
    features=imputer.transform(features)

    # Train the classifier
    model = RandomForestClassifier().fit(features,labels)

    # Save model.
    print('Saving model...')

    final_model={'model':model, 'imputer':imputer,'classes':classes}

    filename = os.path.join(output_directory, 'finalized_model.sav')
    joblib.dump(final_model, filename, protocol=0)
//  [python]
def _get_estimator(pblm, clf_key):
        """
        Returns sklearn classifier
        """
        tup = clf_key.split('-')
        wrap_type = None if len(tup) == 1 else tup[1]
        est_type = tup[0]
        multiclass_wrapper = {
            None: ut.identity,
            'OVR': sklearn.multiclass.OneVsRestClassifier,
            'OVO': sklearn.multiclass.OneVsOneClassifier,
        }[wrap_type]
        est_class = {
            'RF': sklearn.ensemble.RandomForestClassifier,
            'SVC': sklearn.svm.SVC,
            'Logit': sklearn.linear_model.LogisticRegression,
            'MLP': sklearn.neural_network.MLPClassifier,
        }[est_type]

        est_kw1, est_kw2 = pblm._estimator_params(est_type)
        est_params = ut.merge_dicts(est_kw1, est_kw2)

        try:
            from sklearn.preprocessing import Imputer
            ikw = dict(missing_values='NaN', strategy='mean', axis=0)
        except Exception:
            from sklearn.impute import SimpleImputer as Imputer
            ikw = dict(missing_values=float('nan'), strategy='mean')

        # steps = []
        # steps.append((est_type, est_class(**est_params)))
        # if wrap_type is not None:
        #     steps.append((wrap_type, multiclass_wrapper))
        if est_type == 'MLP':
            def clf_partial():
                pipe = sklearn.pipeline.Pipeline([
                    ('inputer', Imputer(**ikw)),
                    # ('scale', sklearn.preprocessing.StandardScaler),
                    ('est', est_class(**est_params)),
                ])
                return multiclass_wrapper(pipe)
        elif est_type == 'Logit':
            def clf_partial():
                pipe = sklearn.pipeline.Pipeline([
                    ('inputer', Imputer(**ikw)),
                    ('est', est_class(**est_params)),
                ])
                return multiclass_wrapper(pipe)
        else:
            def clf_partial():
                return multiclass_wrapper(est_class(**est_params))

        return clf_partial
//  [python]
def encode_features(context):

    imp_constant = sklearn.preprocessing.Imputer(strategy="constant", fill_value="missing")
    ohe = OneHotEncoder()

    imp_ohe = make_pipeline(imp_constant, ohe)
    vect = CountVectorizer()
    imp = sklearn.preprocessing.Imputer()

    ct = make_column_transformer(
        (imp_ohe, ["Embarked", "Sex"]),
        (vect, "Name"),
        (imp, ["Age", "Fare"]),
        remainder="passthrough",
    )

    return ct
//  [python]
def encode_features(context):

    imp_constant = SimpleImputer(strategy="constant", fill_value="missing")
    ohe = OneHotEncoder()

    imp_ohe = make_pipeline(imp_constant, ohe)
    vect = CountVectorizer()
    imp = SimpleImputer()

    ct = make_column_transformer(
        (imp_ohe, ["Embarked", "Sex"]),
        (vect, "Name"),
        (imp, ["Age", "Fare"]),
        remainder="passthrough",
    )

    return ct
//  [python]
def _impute_values(self, features):
        """Impute missing values in a feature set.

        Parameters
        ----------
        features: array-like {n_samples, n_features}
            A feature matrix

        Returns
        -------
        array-like {n_samples, n_features}
        """
        if self.verbosity > 1:
            print("Imputing missing values in feature set")

        if self._fitted_imputer is None:
            self._fitted_imputer = Imputer(strategy="median")
            self._fitted_imputer.fit(features)

        return self._fitted_imputer.transform(features)
//  [python]
def _impute_values(self, features):
        """Impute missing values in a feature set.

        Parameters
        ----------
        features: array-like {n_samples, n_features}
            A feature matrix

        Returns
        -------
        array-like {n_samples, n_features}
        """
        if self.verbosity > 1:
            print("Imputing missing values in feature set")

        if self._fitted_imputer is None:
            self._fitted_imputer = SimpleImputer(strategy="median")
            self._fitted_imputer.fit(features)

        return self._fitted_imputer.transform(features)
//  [python]
def preprocess_data(data_folder='./data'):
    print('preprocessing data')

    df = read_parquet(f'{data_folder}/data.parquet')
    train, _ = train_test_split(df, random_state=43)

    tt_xform = (
        'onehot',
        OneHotEncoder(
            handle_unknown='ignore',
            categories=[[
                'online', 'contactless', 'chip_and_pin', 'manual', 'swipe'
            ]]
        ),
        ['trans_type']
    )

    impute_and_scale = Pipeline([
        ('median_imputer', Imputer(strategy='median')),
        ('interarrival_scaler', RobustScaler())
    ])
    ia_scaler = ('interarrival_scaler', impute_and_scale, ['interarrival'])
    amount_scaler = ('amount_scaler', RobustScaler(), ['amount'])

    all_xforms = ColumnTransformer(
        transformers=([ia_scaler, amount_scaler, tt_xform])
    )
    feature_pipeline = Pipeline([('feature_extraction', all_xforms)])

    feature_columns = [
        'user_id', 'amount', 'trans_type', 'foreign', 'interarrival'
    ]
    feature_pipeline.fit(train[feature_columns])
    dump(feature_pipeline, open('feature_pipeline.joblib', 'wb'))

    print('data processing done')
//  [python]
def preprocess_data(data_folder='./data'):
    print('preprocessing data')

    df = read_parquet(f'{data_folder}/data.parquet')
    train, _ = train_test_split(df, random_state=43)

    tt_xform = (
        'onehot',
        OneHotEncoder(
            handle_unknown='ignore',
            categories=[[
                'online', 'contactless', 'chip_and_pin', 'manual', 'swipe'
            ]]
        ),
        ['trans_type']
    )

    impute_and_scale = Pipeline([
        ('median_imputer', SimpleImputer(strategy='median')),
        ('interarrival_scaler', RobustScaler())
    ])
    ia_scaler = ('interarrival_scaler', impute_and_scale, ['interarrival'])
    amount_scaler = ('amount_scaler', RobustScaler(), ['amount'])

    all_xforms = ColumnTransformer(
        transformers=([ia_scaler, amount_scaler, tt_xform])
    )
    feature_pipeline = Pipeline([('feature_extraction', all_xforms)])

    feature_columns = [
        'user_id', 'amount', 'trans_type', 'foreign', 'interarrival'
    ]
    feature_pipeline.fit(train[feature_columns])
    dump(feature_pipeline, open('feature_pipeline.joblib', 'wb'))

    print('data processing done')
//  [python]
def test_anonymize_pandas_nursery():
    (x_train, y_train), _ = get_nursery_dataset_pd()
    x_train = x_train.astype(str)

    k = 100
    features = ["parents", "has_nurs", "form", "children", "housing", "finance", "social", "health"]
    QI = ["finance", "social", "health"]
    categorical_features = ["parents", "has_nurs", "form", "housing", "finance", "social", "health", 'children']
    # prepare data for DT
    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', Imputer(strategy='constant', fill_value=0))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    model = DecisionTreeClassifier()
    model.fit(encoded, y_train)
    pred = model.predict(encoded)

    anonymizer = Anonymize(k, QI, categorical_features=categorical_features, train_only_QI=True)
    anon = anonymizer.anonymize(ArrayDataset(x_train, pred))

    assert (anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0])
    assert (anon.loc[:, QI].value_counts().min() >= k)
    np.testing.assert_array_equal(anon.drop(QI, axis=1), x_train.drop(QI, axis=1))
//  [python]
def test_anonymize_pandas_nursery():
    (x_train, y_train), _ = get_nursery_dataset_pd()
    x_train = x_train.astype(str)

    k = 100
    features = ["parents", "has_nurs", "form", "children", "housing", "finance", "social", "health"]
    QI = ["finance", "social", "health"]
    categorical_features = ["parents", "has_nurs", "form", "housing", "finance", "social", "health", 'children']
    # prepare data for DT
    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    model = DecisionTreeClassifier()
    model.fit(encoded, y_train)
    pred = model.predict(encoded)

    anonymizer = Anonymize(k, QI, categorical_features=categorical_features, train_only_QI=True)
    anon = anonymizer.anonymize(ArrayDataset(x_train, pred))

    assert (anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0])
    assert (anon.loc[:, QI].value_counts().min() >= k)
    np.testing.assert_array_equal(anon.drop(QI, axis=1), x_train.drop(QI, axis=1))
//  [python]
def test_anonymize_pandas_adult():
    (x_train, y_train), _ = get_adult_dataset_pd()

    k = 100
    features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
    QI = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
          'native-country']
    categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                            'native-country']
    # prepare data for DT
    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', Imputer(strategy='constant', fill_value=0))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    model = DecisionTreeClassifier()
    model.fit(encoded, y_train)
    pred = model.predict(encoded)

    anonymizer = Anonymize(k, QI, categorical_features=categorical_features)
    anon = anonymizer.anonymize(ArrayDataset(x_train, pred, features))

    assert (anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0])
    assert (anon.loc[:, QI].value_counts().min() >= k)
    np.testing.assert_array_equal(anon.drop(QI, axis=1), x_train.drop(QI, axis=1))
//  [python]
def test_anonymize_pandas_adult():
    (x_train, y_train), _ = get_adult_dataset_pd()

    k = 100
    features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
    QI = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
          'native-country']
    categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                            'native-country']
    # prepare data for DT
    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    model = DecisionTreeClassifier()
    model.fit(encoded, y_train)
    pred = model.predict(encoded)

    anonymizer = Anonymize(k, QI, categorical_features=categorical_features)
    anon = anonymizer.anonymize(ArrayDataset(x_train, pred, features))

    assert (anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0])
    assert (anon.loc[:, QI].value_counts().min() >= k)
    np.testing.assert_array_equal(anon.drop(QI, axis=1), x_train.drop(QI, axis=1))
//  [python]
def fit(
        self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
    ) -> "NumericalImputation":
        import sklearn.preprocessing

        self.preprocessor = sklearn.preprocessing.Imputer(
            strategy=self.strategy, copy=False
        )
        self.preprocessor.fit(X)
        return self
//  [python]
def fit(
        self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
    ) -> "NumericalImputation":
        import sklearn.impute

        self.preprocessor = sklearn.impute.SimpleImputer(
            strategy=self.strategy, copy=False
        )
        self.preprocessor.fit(X)
        return self
//  [python]
def inbuiltColumnsTransformer(data):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        from sklearn.preprocessing import Imputer
        
        category_cols, numerical_cols = ModelProcessor.num_cat_variables(data)
        category_map = ModelProcessor.gen_category_map(data, category_cols)

        
        ordinal_features = [x for x in range(len(data.columns)) if x not in list(category_map.keys())]
        ordinal_transformer = Pipeline(steps=[('imputer', Imputer(strategy='median')),
                                    ('scaler', StandardScaler())])
        
        categorical_features = list(category_map.keys())
        categorical_transformer = Pipeline(steps=[('imputer', Imputer(strategy='most_frequent')),
                                        ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
        
        #pass the pipeline into the transformer
        preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                            ('cat', categorical_transformer, categorical_features)])
        #preprocessor.fit(data)
        return preprocessor
//  [python]
def inbuiltColumnsTransformer(data):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        
        category_cols, numerical_cols = ModelProcessor.num_cat_variables(data)
        category_map = ModelProcessor.gen_category_map(data, category_cols)

        
        ordinal_features = [x for x in range(len(data.columns)) if x not in list(category_map.keys())]
        ordinal_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),
                                    ('scaler', StandardScaler())])
        
        categorical_features = list(category_map.keys())
        categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')),
                                        ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
        
        #pass the pipeline into the transformer
        preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                            ('cat', categorical_transformer, categorical_features)])
        #preprocessor.fit(data)
        return preprocessor
//  [python]
def test_mlforecast_on_m5(regressor, pd_m5_dataset, benchmark):
    from joblib import cpu_count
    from mlforecast import MLForecast
    from mlforecast.target_transforms import LocalStandardScaler
    from sklearn.preprocessing import Imputer

    X_y_train, _, X_test, fh, freq, lags, entity_col, time_col = pd_m5_dataset
    _, regressor_cls = regressor

    # NOTE: We create a sklearn pipeline with imputer because
    # lags > 3 raises input X contains NaN, infinity or a
    # value too large for dtype('float64').

    pipeline = Pipeline([("impute", Imputer()), ("regressor", regressor_cls())])

    def fit_predict():
        forecaster = MLForecast(
            models=[pipeline],
            freq=freq,
            lags=list(range(lags)),
            num_threads=cpu_count(),
            target_transforms=[LocalStandardScaler()],
        )
        forecaster.fit(
            X_y_train.reset_index(),
            id_col=entity_col,
            time_col=time_col,
            target_col="quantity_sold",
            keep_last_n=lags,
        )
        y_pred = forecaster.predict(fh, dynamic_dfs=[X_test])
        return y_pred

    benchmark(fit_predict)
//  [python]
def test_mlforecast_on_m5(regressor, pd_m5_dataset, benchmark):
    from joblib import cpu_count
    from mlforecast import MLForecast
    from mlforecast.target_transforms import LocalStandardScaler

    X_y_train, _, X_test, fh, freq, lags, entity_col, time_col = pd_m5_dataset
    _, regressor_cls = regressor

    # NOTE: We create a sklearn pipeline with imputer because
    # lags > 3 raises input X contains NaN, infinity or a
    # value too large for dtype('float64').

    pipeline = Pipeline([("impute", SimpleImputer()), ("regressor", regressor_cls())])

    def fit_predict():
        forecaster = MLForecast(
            models=[pipeline],
            freq=freq,
            lags=list(range(lags)),
            num_threads=cpu_count(),
            target_transforms=[LocalStandardScaler()],
        )
        forecaster.fit(
            X_y_train.reset_index(),
            id_col=entity_col,
            time_col=time_col,
            target_col="quantity_sold",
            keep_last_n=lags,
        )
        y_pred = forecaster.predict(fh, dynamic_dfs=[X_test])
        return y_pred

    benchmark(fit_predict)
//  [python]
def columnsTransformer(self):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        from sklearn.preprocessing import Imputer
        
        category_cols, numerical_cols = self.num_cat_variables(self.input_data)
        category_map = self.gen_category_map(self.input_data, category_cols)

        if self.ct == None:
            pass
        elif self.ct == "default":
            ordinal_features = [x for x in range(len(self.input_data.columns)) if x not in list(category_map.keys())]
            ordinal_transformer = Pipeline(steps=[('imputer', Imputer(strategy='median')),
                                      ('scaler', StandardScaler())])
            
            categorical_features = list(category_map.keys())
            categorical_transformer = Pipeline(steps=[('imputer', Imputer(strategy='most_frequent')),
                                          ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
            
            #pass the pipeline into the transformer
            preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                               ('cat', categorical_transformer, categorical_features)])
            preprocessor.fit(self.input_data)
            return preprocessor
        else:
            transformerFunction = ct.fit(self.input_data)
            #transformed_xtest = ct.transform(x_test)
            return transformerFunction
//  [python]
def columnsTransformer(self):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        
        category_cols, numerical_cols = self.num_cat_variables(self.input_data)
        category_map = self.gen_category_map(self.input_data, category_cols)

        if self.ct == None:
            pass
        elif self.ct == "default":
            ordinal_features = [x for x in range(len(self.input_data.columns)) if x not in list(category_map.keys())]
            ordinal_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),
                                      ('scaler', StandardScaler())])
            
            categorical_features = list(category_map.keys())
            categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')),
                                          ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
            
            #pass the pipeline into the transformer
            preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                               ('cat', categorical_transformer, categorical_features)])
            preprocessor.fit(self.input_data)
            return preprocessor
        else:
            transformerFunction = ct.fit(self.input_data)
            #transformed_xtest = ct.transform(x_test)
            return transformerFunction
//  [python]
def _sklearn_pipeline(self):
        model = sklearn.neural_network.MLPClassifier(activation="tanh", max_iter=2000, hidden_layer_sizes=(10,))
        numerical_transformer = sklearn.pipeline.Pipeline(
            [
                ("imputer", sklearn.preprocessing.Imputer(strategy="constant", fill_value=0)),
            ]
        )
        preprocess = sklearn.compose.ColumnTransformer(
            [
                ("numerical", numerical_transformer[0, 1]),
            ]
        )
        pipeline = sklearn.pipeline.Pipeline(
            [
                ("preprocess", preprocess),
                ("model", model),
            ]
        )

        pipeline.fit(
            np.array([[0, np.nan], [np.nan, 1], [1, 0], [1, 1]]),  # NaN will be replaced with 0
            np.array([0, 1, 1, 0]),
        )

        return pipeline, sklearn.pipeline.Pipeline
//  [python]
def _sklearn_pipeline(self):
        model = sklearn.neural_network.MLPClassifier(activation="tanh", max_iter=2000, hidden_layer_sizes=(10,))
        numerical_transformer = sklearn.pipeline.Pipeline(
            [
                ("imputer", sklearn.impute.SimpleImputer(strategy="constant", fill_value=0)),
            ]
        )
        preprocess = sklearn.compose.ColumnTransformer(
            [
                ("numerical", numerical_transformer[0, 1]),
            ]
        )
        pipeline = sklearn.pipeline.Pipeline(
            [
                ("preprocess", preprocess),
                ("model", model),
            ]
        )

        pipeline.fit(
            np.array([[0, np.nan], [np.nan, 1], [1, 0], [1, 1]]),  # NaN will be replaced with 0
            np.array([0, 1, 1, 0]),
        )

        return pipeline, sklearn.pipeline.Pipeline
//  [python]
categorical_encoder = OrdinalEncoder(
    handle_unknown="use_encoded_value", unknown_value=-1,
)
numerical_pipe = Imputer(strategy="mean")
//  [python]
categorical_encoder = OrdinalEncoder(
    handle_unknown="use_encoded_value", unknown_value=-1,
)
numerical_pipe = SimpleImputer(strategy="mean")
//  [python]
def impute_and_scale(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """
    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        imputer = Imputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """
    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        imputer = SimpleImputer(strategy=imputing)
        mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def get_model_kmeans():
    print("Importing benign.csv data...")
    benign_h2o = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/benign.csv"))

    benign_sci = np.genfromtxt(pyunit_utils.locate("smalldata/logreg/benign.csv"), delimiter=",")
    # Impute missing values with column mean
    imp = Imputer(missing_values=np.nan, strategy="mean")
    benign_sci = imp.fit_transform(benign_sci)

    for i in range(2,7):
        km_h2o = H2OKMeansEstimator(k=i)
        km_h2o.train(x=list(range(benign_h2o.ncol)), training_frame=benign_h2o)
        km_h2o.show()
        model = h2o.get_model(km_h2o._id)
        model.show()

        km_sci = KMeans(n_clusters=i, init='k-means++', n_init=1)
        km_sci.fit(benign_sci)
        print("sckit centers")
        print(km_sci.cluster_centers_)
//  [python]
def get_model_kmeans():
    print("Importing benign.csv data...")
    benign_h2o = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/benign.csv"))

    benign_sci = np.genfromtxt(pyunit_utils.locate("smalldata/logreg/benign.csv"), delimiter=",")
    # Impute missing values with column mean
    imp = SimpleImputer(missing_values=np.nan, strategy="mean")
    benign_sci = imp.fit_transform(benign_sci)

    for i in range(2,7):
        km_h2o = H2OKMeansEstimator(k=i)
        km_h2o.train(x=list(range(benign_h2o.ncol)), training_frame=benign_h2o)
        km_h2o.show()
        model = h2o.get_model(km_h2o._id)
        model.show()

        km_sci = KMeans(n_clusters=i, init='k-means++', n_init=1)
        km_sci.fit(benign_sci)
        print("sckit centers")
        print(km_sci.cluster_centers_)
//  [python]
train_df["review"], test_df["review"] = train_df["review"].apply(decode_html), test_df[
    "review"
].apply(decode_html)

## Dropped the original date column and removed the useless column
train_df, test_df = [
    df.drop("date", axis=1).drop(df.columns[0], axis=1) for df in (train_df, test_df)
]

## Handling the missing values and assigning old column names
train_imp, test_imp = [
    pd.DataFrame(
        Imputer(strategy="most_frequent").fit_transform(df), columns=df.columns
    )
    for df in (train_df, test_df)
]
//  [python]
train_df["review"], test_df["review"] = train_df["review"].apply(decode_html), test_df[
    "review"
].apply(decode_html)

## Dropped the original date column and removed the useless column
train_df, test_df = [
    df.drop("date", axis=1).drop(df.columns[0], axis=1) for df in (train_df, test_df)
]

## Handling the missing values and assigning old column names
train_imp, test_imp = [
    pd.DataFrame(
        SimpleImputer(strategy="most_frequent").fit_transform(df), columns=df.columns
    )
    for df in (train_df, test_df)
]
//  [python]
X[np.isinf(X)] = np.nan
imp = Imputer(missing_values=np.nan, strategy='mean').fit(X)
//  [python]
X[np.isinf(X)] = np.nan
imp = SimpleImputer(missing_values=np.nan, strategy='mean').fit(X)
//  [python]
def replace_missing_mean_median_mode(
    x_train, x_test=None, list_of_cols=[], strategy=""
):
    """
    Replaces missing values in every numeric column with the mean, median or mode of that column specified by strategy.

    Mean: Average value of the column. Effected by outliers.
    Median: Middle value of a list of numbers. Equal to the mean if x_train follows normal distribution. Not effected much by anomalies.
    Mode: Most common number in a list of numbers.
    
    Parameters
    ----------
    x_train: Dataframe or array like - 2d
        Dataset

    x_test: Dataframe or array like - 2d
        Testing dataset, by default None.

    list_of_cols : list, optional
        A list of specific columns to apply this technique to
        If `list_of_cols` is not provided, the strategy will be
        applied to all numeric columns., by default []

    strategy : str
        Strategy for replacing missing values.
        Can be either "mean", "median" or "most_frequent"
    
    Returns
    -------
    Dataframe, *Dataframe
        Transformed dataframe with rows with a missing values in a specific column are missing

    Returns 2 Dataframes test if x_test is provided.  
    """

    if strategy != "most_frequent":
        list_of_cols = _numeric_input_conditions(list_of_cols, x_train)
    else:
        list_of_cols = _get_columns(list_of_cols, x_train)

    imp = Imputer(strategy=strategy)

    fit_data = imp.fit_transform(x_train[list_of_cols])
    fit_df = pd.DataFrame(fit_data, columns=list_of_cols)
    x_train = drop_replace_columns(x_train, list_of_cols, fit_df)

    if x_test is not None:
        fit_x_test = imp.transform(x_test[list_of_cols])
        fit_test_df = pd.DataFrame(fit_x_test, columns=list_of_cols)
        x_test = drop_replace_columns(x_test, list_of_cols, fit_test_df)

    return x_train, x_test
//  [python]
def replace_missing_mean_median_mode(
    x_train, x_test=None, list_of_cols=[], strategy=""
):
    """
    Replaces missing values in every numeric column with the mean, median or mode of that column specified by strategy.

    Mean: Average value of the column. Effected by outliers.
    Median: Middle value of a list of numbers. Equal to the mean if x_train follows normal distribution. Not effected much by anomalies.
    Mode: Most common number in a list of numbers.
    
    Parameters
    ----------
    x_train: Dataframe or array like - 2d
        Dataset

    x_test: Dataframe or array like - 2d
        Testing dataset, by default None.

    list_of_cols : list, optional
        A list of specific columns to apply this technique to
        If `list_of_cols` is not provided, the strategy will be
        applied to all numeric columns., by default []

    strategy : str
        Strategy for replacing missing values.
        Can be either "mean", "median" or "most_frequent"
    
    Returns
    -------
    Dataframe, *Dataframe
        Transformed dataframe with rows with a missing values in a specific column are missing

    Returns 2 Dataframes test if x_test is provided.  
    """

    if strategy != "most_frequent":
        list_of_cols = _numeric_input_conditions(list_of_cols, x_train)
    else:
        list_of_cols = _get_columns(list_of_cols, x_train)

    imp = SimpleImputer(strategy=strategy)

    fit_data = imp.fit_transform(x_train[list_of_cols])
    fit_df = pd.DataFrame(fit_data, columns=list_of_cols)
    x_train = drop_replace_columns(x_train, list_of_cols, fit_df)

    if x_test is not None:
        fit_x_test = imp.transform(x_test[list_of_cols])
        fit_test_df = pd.DataFrame(fit_x_test, columns=list_of_cols)
        x_test = drop_replace_columns(x_test, list_of_cols, fit_test_df)

    return x_train, x_test
//  [python]
def generate_pipeline(x_train, numerical_model, model_config):
    """Generate the data pipeline
    Arguments
    x_train: independent features which need to be transformed
    numerical_model: parameter of the numerical transformer
    model_config: configurations of the model

    Return
    pipeline object used to process all features
    """

    # Categrical features to pass down the categorical pipeline
    categorical_features = x_train.select_dtypes("object").columns.to_list()

    # Numerical features to pass down the numerical pipeline
    numerical_features = x_train.select_dtypes("int64").columns.to_list()

    # Defining the steps in the categorical pipeline
    categorical_pipeline = Pipeline(steps=[('cat_selector', FeatureSelector(categorical_features)),
                                           ('imputer_cat', Imputer(
                                               strategy="most_frequent")),
                                           ('cat_transformer', CategoricalTransformer(
                                               colnames=categorical_features)),
                                           # ('cat_encoder','passthrough'
                                           ('cat_encoder', OneHotEncoder(
                                               sparse=False, drop="first"))
                                           ]
                                    )

    # Defining the steps in the numerical pipeline
    numerical_pipeline = Pipeline(
        steps=[
            ('num_selector', FeatureSelector(numerical_features)), ('imputer_num', Imputer(
                strategy="median")), ('num_transformer', NumericalTransformer(
                    numerical_model, colnames=numerical_features))])

    # Combining numerical and categorical piepline into one full big pipeline horizontally
    # using FeatureUnion
    full_pipeline_preprocessing = FeatureUnion(
        transformer_list=[
            ('cat_pipeline', categorical_pipeline), ('num_pipeline', numerical_pipeline)])

    # The full pipeline
    pipe = Pipeline(
        steps=[
            ('full_pipeline',
             full_pipeline_preprocessing),
            ("classifier",
             XGBClassifier(
                 **model_config))])
    return pipe
//  [python]
def generate_pipeline(x_train, numerical_model, model_config):
    """Generate the data pipeline
    Arguments
    x_train: independent features which need to be transformed
    numerical_model: parameter of the numerical transformer
    model_config: configurations of the model

    Return
    pipeline object used to process all features
    """

    # Categrical features to pass down the categorical pipeline
    categorical_features = x_train.select_dtypes("object").columns.to_list()

    # Numerical features to pass down the numerical pipeline
    numerical_features = x_train.select_dtypes("int64").columns.to_list()

    # Defining the steps in the categorical pipeline
    categorical_pipeline = Pipeline(steps=[('cat_selector', FeatureSelector(categorical_features)),
                                           ('imputer_cat', SimpleImputer(
                                               strategy="most_frequent")),
                                           ('cat_transformer', CategoricalTransformer(
                                               colnames=categorical_features)),
                                           # ('cat_encoder','passthrough'
                                           ('cat_encoder', OneHotEncoder(
                                               sparse=False, drop="first"))
                                           ]
                                    )

    # Defining the steps in the numerical pipeline
    numerical_pipeline = Pipeline(
        steps=[
            ('num_selector', FeatureSelector(numerical_features)), ('imputer_num', SimpleImputer(
                strategy="median")), ('num_transformer', NumericalTransformer(
                    numerical_model, colnames=numerical_features))])

    # Combining numerical and categorical piepline into one full big pipeline horizontally
    # using FeatureUnion
    full_pipeline_preprocessing = FeatureUnion(
        transformer_list=[
            ('cat_pipeline', categorical_pipeline), ('num_pipeline', numerical_pipeline)])

    # The full pipeline
    pipe = Pipeline(
        steps=[
            ('full_pipeline',
             full_pipeline_preprocessing),
            ("classifier",
             XGBClassifier(
                 **model_config))])
    return pipe