jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
features_train = data_train.loc[:, data_train.columns != target_column_name]
labels_train = data_train[target_column_name]
features_test = data_test.loc[:, data_test.columns != target_column_name]
labels_test = data_test[target_column_name]

if args.library == 'pytorch' or args.library == 'sklearn':
	from sklearn.preprocessing import StandardScaler
	from sklearn.compose import ColumnTransformer
	from sklearn.pipeline import Pipeline
	from sklearn.impute import SimpleImputer
	from sklearn.preprocessing import StandardScaler, OneHotEncoder
	numeric_features = features_train.select_dtypes(
		include=[np.float64, np.int64]
	).columns
	numeric_transformer = Pipeline(steps=[
		('imputer', SimpleImputer(strategy='median')),
		('scaler', StandardScaler())
	])
	categorical_features = features_train.select_dtypes(
		include=['category']
	).columns
	categorical_transformer = Pipeline(
		steps=[
			('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
			('onehot', OneHotEncoder(handle_unknown='ignore'))
	])
	preprocessor = ColumnTransformer(
		transformers=[
			('num', numeric_transformer, numeric_features),
			('cat', categorical_transformer, categorical_features)
	])
	features_train = preprocessor.fit_transform(features_train)
	features_test = preprocessor.transform(features_test)
	print(features_train.shape)
//  [python]
def test_estimator_html_repr_pipeline():
    num_trans = Pipeline(
        steps=[("pass", "passthrough"), ("imputer", Imputer(strategy="median"))]
    )

    cat_trans = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", missing_values="empty")),
            ("one-hot", OneHotEncoder(drop="first")),
        ]
    )

    preprocess = ColumnTransformer(
        [
            ("num", num_trans, ["a", "b", "c", "d", "e"]),
            ("cat", cat_trans, [0, 1, 2, 3]),
        ]
    )

    feat_u = FeatureUnion(
        [
            ("pca", PCA(n_components=1)),
            (
                "tsvd",
                Pipeline(
                    [
                        ("first", TruncatedSVD(n_components=3)),
                        ("select", SelectPercentile()),
                    ]
                ),
            ),
        ]
    )

    clf = VotingClassifier(
        [
            ("lr", LogisticRegression(solver="lbfgs", random_state=1)),
            ("mlp", MLPClassifier(alpha=0.001)),
        ]
    )

    pipe = Pipeline(
        [("preprocessor", preprocess), ("feat_u", feat_u), ("classifier", clf)]
    )
    html_output = estimator_html_repr(pipe)

    # top level estimators show estimator with changes
    assert html.escape(str(pipe)) in html_output
    for _, est in pipe.steps:
        assert (
            '<div class="sk-toggleable__content"><pre>' + html.escape(str(est))
        ) in html_output

    # low level estimators do not show changes
    with config_context(print_changed_only=True):
        assert html.escape(str(num_trans["pass"])) in html_output
        assert "passthrough</label>" in html_output
        assert html.escape(str(num_trans["imputer"])) in html_output

        for _, _, cols in preprocess.transformers:
            assert f"<pre>{html.escape(str(cols))}</pre>" in html_output

        # feature union
        for name, _ in feat_u.transformer_list:
            assert f"<label>{html.escape(name)}</label>" in html_output

        pca = feat_u.transformer_list[0][1]
        assert f"<pre>{html.escape(str(pca))}</pre>" in html_output

        tsvd = feat_u.transformer_list[1][1]
        first = tsvd["first"]
        select = tsvd["select"]
        assert f"<pre>{html.escape(str(first))}</pre>" in html_output
        assert f"<pre>{html.escape(str(select))}</pre>" in html_output

        # voting classifier
        for name, est in clf.estimators:
            assert f"<label>{html.escape(name)}</label>" in html_output
            assert f"<pre>{html.escape(str(est))}</pre>" in html_output
//  [python]
def test_estimator_html_repr_pipeline():
    num_trans = Pipeline(
        steps=[("pass", "passthrough"), ("imputer", SimpleImputer(strategy="median"))]
    )

    cat_trans = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", missing_values="empty")),
            ("one-hot", OneHotEncoder(drop="first")),
        ]
    )

    preprocess = ColumnTransformer(
        [
            ("num", num_trans, ["a", "b", "c", "d", "e"]),
            ("cat", cat_trans, [0, 1, 2, 3]),
        ]
    )

    feat_u = FeatureUnion(
        [
            ("pca", PCA(n_components=1)),
            (
                "tsvd",
                Pipeline(
                    [
                        ("first", TruncatedSVD(n_components=3)),
                        ("select", SelectPercentile()),
                    ]
                ),
            ),
        ]
    )

    clf = VotingClassifier(
        [
            ("lr", LogisticRegression(solver="lbfgs", random_state=1)),
            ("mlp", MLPClassifier(alpha=0.001)),
        ]
    )

    pipe = Pipeline(
        [("preprocessor", preprocess), ("feat_u", feat_u), ("classifier", clf)]
    )
    html_output = estimator_html_repr(pipe)

    # top level estimators show estimator with changes
    assert html.escape(str(pipe)) in html_output
    for _, est in pipe.steps:
        assert (
            '<div class="sk-toggleable__content"><pre>' + html.escape(str(est))
        ) in html_output

    # low level estimators do not show changes
    with config_context(print_changed_only=True):
        assert html.escape(str(num_trans["pass"])) in html_output
        assert "passthrough</label>" in html_output
        assert html.escape(str(num_trans["imputer"])) in html_output

        for _, _, cols in preprocess.transformers:
            assert f"<pre>{html.escape(str(cols))}</pre>" in html_output

        # feature union
        for name, _ in feat_u.transformer_list:
            assert f"<label>{html.escape(name)}</label>" in html_output

        pca = feat_u.transformer_list[0][1]
        assert f"<pre>{html.escape(str(pca))}</pre>" in html_output

        tsvd = feat_u.transformer_list[1][1]
        first = tsvd["first"]
        select = tsvd["select"]
        assert f"<pre>{html.escape(str(first))}</pre>" in html_output
        assert f"<pre>{html.escape(str(select))}</pre>" in html_output

        # voting classifier
        for name, est in clf.estimators:
            assert f"<label>{html.escape(name)}</label>" in html_output
            assert f"<pre>{html.escape(str(est))}</pre>" in html_output
//  [python]
def test_estimator_html_repr_pipeline():
    num_trans = Pipeline(steps=[
        ('pass', 'passthrough'),
        ('imputer', Imputer(strategy='median'))
    ])

    cat_trans = Pipeline(steps=[
        ('imputer', Imputer(strategy='constant',
                                  missing_values='empty')),
        ('one-hot', OneHotEncoder(drop='first'))
    ])

    preprocess = ColumnTransformer([
        ('num', num_trans, ['a', 'b', 'c', 'd', 'e']),
        ('cat', cat_trans, [0, 1, 2, 3])
    ])

    feat_u = FeatureUnion([
            ('pca', PCA(n_components=1)),
            ('tsvd', Pipeline([('first', TruncatedSVD(n_components=3)),
                               ('select', SelectPercentile())]))
    ])

    clf = VotingClassifier([
        ('lr', LogisticRegression(solver='lbfgs', random_state=1)),
        ('mlp', MLPClassifier(alpha=0.001))
    ])

    pipe = Pipeline([
        ('preprocessor', preprocess), ('feat_u', feat_u), ('classifier', clf)
    ])
    html_output = estimator_html_repr(pipe)

    # top level estimators show estimator with changes
    assert str(pipe) in html_output
    for _, est in pipe.steps:
        assert (f"<div class=\"sk-toggleable__content\">"
                f"<pre>{str(est)}") in html_output

    # low level estimators do not show changes
    with config_context(print_changed_only=True):
        assert str(num_trans['pass']) in html_output
        assert 'passthrough</label>' in html_output
        assert str(num_trans['imputer']) in html_output

        for _, _, cols in preprocess.transformers:
            assert f"<pre>{cols}</pre>" in html_output

        # feature union
        for name, _ in feat_u.transformer_list:
            assert f"<label>{name}</label>" in html_output

        pca = feat_u.transformer_list[0][1]
        assert f"<pre>{str(pca)}</pre>" in html_output

        tsvd = feat_u.transformer_list[1][1]
        first = tsvd['first']
        select = tsvd['select']
        assert f"<pre>{str(first)}</pre>" in html_output
        assert f"<pre>{str(select)}</pre>" in html_output

        # voting classifer
        for name, est in clf.estimators:
            assert f"<label>{name}</label>" in html_output
            assert f"<pre>{str(est)}</pre>" in html_output
//  [python]
def test_estimator_html_repr_pipeline():
    num_trans = Pipeline(steps=[
        ('pass', 'passthrough'),
        ('imputer', SimpleImputer(strategy='median'))
    ])

    cat_trans = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant',
                                  missing_values='empty')),
        ('one-hot', OneHotEncoder(drop='first'))
    ])

    preprocess = ColumnTransformer([
        ('num', num_trans, ['a', 'b', 'c', 'd', 'e']),
        ('cat', cat_trans, [0, 1, 2, 3])
    ])

    feat_u = FeatureUnion([
            ('pca', PCA(n_components=1)),
            ('tsvd', Pipeline([('first', TruncatedSVD(n_components=3)),
                               ('select', SelectPercentile())]))
    ])

    clf = VotingClassifier([
        ('lr', LogisticRegression(solver='lbfgs', random_state=1)),
        ('mlp', MLPClassifier(alpha=0.001))
    ])

    pipe = Pipeline([
        ('preprocessor', preprocess), ('feat_u', feat_u), ('classifier', clf)
    ])
    html_output = estimator_html_repr(pipe)

    # top level estimators show estimator with changes
    assert str(pipe) in html_output
    for _, est in pipe.steps:
        assert (f"<div class=\"sk-toggleable__content\">"
                f"<pre>{str(est)}") in html_output

    # low level estimators do not show changes
    with config_context(print_changed_only=True):
        assert str(num_trans['pass']) in html_output
        assert 'passthrough</label>' in html_output
        assert str(num_trans['imputer']) in html_output

        for _, _, cols in preprocess.transformers:
            assert f"<pre>{cols}</pre>" in html_output

        # feature union
        for name, _ in feat_u.transformer_list:
            assert f"<label>{name}</label>" in html_output

        pca = feat_u.transformer_list[0][1]
        assert f"<pre>{str(pca)}</pre>" in html_output

        tsvd = feat_u.transformer_list[1][1]
        first = tsvd['first']
        select = tsvd['select']
        assert f"<pre>{str(first)}</pre>" in html_output
        assert f"<pre>{str(select)}</pre>" in html_output

        # voting classifer
        for name, est in clf.estimators:
            assert f"<label>{name}</label>" in html_output
            assert f"<pre>{str(est)}</pre>" in html_output
//  [python]
def test_simple_imputer_float_inputs_int_mean(self):
        model = Imputer(strategy="mean", fill_value="nan")
        data = [[1, 2], [np.nan, 3], [7, 6], [8, np.nan]]
        model.fit(data)

        try:
            convert_sklearn(
                model,
                "scikit-learn simple imputer",
                [("input", Int64TensorType([None, 2]))],
                target_opset=TARGET_OPSET,
            )
        except RuntimeError as e:
            assert "nan values are replaced by float" in str(e)
//  [python]
def test_simple_imputer_float_inputs_int_mean(self):
        model = SimpleImputer(strategy="mean", fill_value="nan")
        data = [[1, 2], [np.nan, 3], [7, 6], [8, np.nan]]
        model.fit(data)

        try:
            convert_sklearn(
                model,
                "scikit-learn simple imputer",
                [("input", Int64TensorType([None, 2]))],
                target_opset=TARGET_OPSET,
            )
        except RuntimeError as e:
            assert "nan values are replaced by float" in str(e)
//  [python]
def test_simple_imputer_string_inputs_int_mostf(self):
        model = Imputer(
            strategy="most_frequent", fill_value="nan", missing_values=""
        )
        data = [["s1", "s2"], ["", "s3"], ["s7", "s6"], ["s8", ""]]
        model.fit(data)
        model_onnx = convert_sklearn(
            model,
            "scikit-learn simple imputer",
            [("input", StringTensorType([None, 2]))],
            target_opset=TARGET_OPSET,
        )
        self.assertIn("ai.onnx.ml", str(model_onnx))
        self.assertTrue(model_onnx.graph.node is not None)
        self.assertEqual(len(model_onnx.graph.output), 1)
        self._check_outputs_strings(model, model_onnx, data)
//  [python]
def test_simple_imputer_string_inputs_int_mostf(self):
        model = SimpleImputer(
            strategy="most_frequent", fill_value="nan", missing_values=""
        )
        data = [["s1", "s2"], ["", "s3"], ["s7", "s6"], ["s8", ""]]
        model.fit(data)
        model_onnx = convert_sklearn(
            model,
            "scikit-learn simple imputer",
            [("input", StringTensorType([None, 2]))],
            target_opset=TARGET_OPSET,
        )
        self.assertIn("ai.onnx.ml", str(model_onnx))
        self.assertTrue(model_onnx.graph.node is not None)
        self.assertEqual(len(model_onnx.graph.output), 1)
        self._check_outputs_strings(model, model_onnx, data)
//  [python]
def test_impute_median_500k_sklearn(pddf_with_nulls, benchmark):

    df: pd.DataFrame = pddf_with_nulls
    si = Imputer(strategy="median")
    _ = benchmark(
        si.fit_transform, df
    ) # The fact that they are the same is tested in test_impute_median_500k_dsds
    assert 1 == 1
//  [python]
def test_impute_median_500k_sklearn(pddf_with_nulls, benchmark):

    df: pd.DataFrame = pddf_with_nulls
    si = SimpleImputer(strategy="median")
    _ = benchmark(
        si.fit_transform, df
    ) # The fact that they are the same is tested in test_impute_median_500k_dsds
    assert 1 == 1
//  [python]
def test_impute_median_500k_dsds(pldf_with_nulls, benchmark):

    df: pl.DataFrame = pldf_with_nulls
    imputed_pl = benchmark(
        t.impute, df, ["c1", "c2", "c3"]
    )

    si = Imputer(strategy="median")
    imputed_pd = pl.from_pandas(si.fit_transform(df.to_pandas()))
    assert_frame_equal(imputed_pl, imputed_pd)
//  [python]
def test_impute_median_500k_dsds(pldf_with_nulls, benchmark):

    df: pl.DataFrame = pldf_with_nulls
    imputed_pl = benchmark(
        t.impute, df, ["c1", "c2", "c3"]
    )

    si = SimpleImputer(strategy="median")
    imputed_pd = pl.from_pandas(si.fit_transform(df.to_pandas()))
    assert_frame_equal(imputed_pl, imputed_pd)
//  [python]
def cat_process_nans(X: ArrayDict, policy: Optional[CatNanPolicy]) -> ArrayDict:
    assert X is not None
    nan_masks = {k: v == CAT_MISSING_VALUE for k, v in X.items()}
    if any(x.any() for x in nan_masks.values()):  # type: ignore[code]
        if policy is None:
            X_new = X
        elif policy == 'most_frequent':
            imputer = Imputer(missing_values=CAT_MISSING_VALUE, strategy=policy)  # type: ignore[code]
            imputer.fit(X['train'])
            X_new = {k: cast(np.ndarray, imputer.transform(v)) for k, v in X.items()}
        else:
            util.raise_unknown('categorical NaN policy', policy)
    else:
        assert policy is None
        X_new = X
    return X_new
//  [python]
def cat_process_nans(X: ArrayDict, policy: Optional[CatNanPolicy]) -> ArrayDict:
    assert X is not None
    nan_masks = {k: v == CAT_MISSING_VALUE for k, v in X.items()}
    if any(x.any() for x in nan_masks.values()):  # type: ignore[code]
        if policy is None:
            X_new = X
        elif policy == 'most_frequent':
            imputer = SimpleImputer(missing_values=CAT_MISSING_VALUE, strategy=policy)  # type: ignore[code]
            imputer.fit(X['train'])
            X_new = {k: cast(np.ndarray, imputer.transform(v)) for k, v in X.items()}
        else:
            util.raise_unknown('categorical NaN policy', policy)
    else:
        assert policy is None
        X_new = X
    return X_new
//  [python]
def test_simple_imputer_string_inputs_int_mostf_default(self):
        model = Imputer(strategy="most_frequent", missing_values="")
        data = pd.DataFrame(
            [["s1", "s2"], ["s1", "s2"], ["", "s3"], ["s7", "s6"], ["s8", ""]]
        )
        model.fit(data)
        model_onnx = convert_sklearn(
            model,
            "scikit-learn simple imputer",
            [("input", StringTensorType([None, 2]))],
            target_opset=TARGET_OPSET,
        )
        self.assertIn("ai.onnx.ml", str(model_onnx))
        self.assertTrue(model_onnx.graph.node is not None)
        self.assertEqual(len(model_onnx.graph.output), 1)
        self._check_outputs_strings(model, model_onnx, data)
//  [python]
def test_simple_imputer_string_inputs_int_mostf_default(self):
        model = SimpleImputer(strategy="most_frequent", missing_values="")
        data = pd.DataFrame(
            [["s1", "s2"], ["s1", "s2"], ["", "s3"], ["s7", "s6"], ["s8", ""]]
        )
        model.fit(data)
        model_onnx = convert_sklearn(
            model,
            "scikit-learn simple imputer",
            [("input", StringTensorType([None, 2]))],
            target_opset=TARGET_OPSET,
        )
        self.assertIn("ai.onnx.ml", str(model_onnx))
        self.assertTrue(model_onnx.graph.node is not None)
        self.assertEqual(len(model_onnx.graph.output), 1)
        self._check_outputs_strings(model, model_onnx, data)
//  [python]
def get_estimator():

    categorical_cols = ['Sex', 'Pclass', 'Embarked']
    numerical_cols = ['Age', 'SibSp', 'Parch', 'Fare']

    preprocessor = make_column_transformer(
        (OneHotEncoder(handle_unknown='ignore'), categorical_cols),
        (Imputer(strategy='constant', fill_value=-1), numerical_cols),
    )

    pipeline = Pipeline([
        ('transformer', preprocessor),
        ('classifier', LogisticRegression()),
    ])

    return pipeline
//  [python]
def get_estimator():

    categorical_cols = ['Sex', 'Pclass', 'Embarked']
    numerical_cols = ['Age', 'SibSp', 'Parch', 'Fare']

    preprocessor = make_column_transformer(
        (OneHotEncoder(handle_unknown='ignore'), categorical_cols),
        (SimpleImputer(strategy='constant', fill_value=-1), numerical_cols),
    )

    pipeline = Pipeline([
        ('transformer', preprocessor),
        ('classifier', LogisticRegression()),
    ])

    return pipeline
//  [python]
def simpleMethod(self):
        """ Get imputed data from scikit SimpleImputer methods
        """
        print("What's goining on? -----linear")
        series_result = Imputer(strategy=self.method, missing_values = np.nan).fit_transform(self.data)
        result = self.makeDF(series_result)
        return result
//  [python]
def simpleMethod(self):
        """ Get imputed data from scikit SimpleImputer methods
        """
        print("What's goining on? -----linear")
        series_result = SimpleImputer(strategy=self.method, missing_values = np.nan).fit_transform(self.data)
        result = self.makeDF(series_result)
        return result
//  [python]
def fillMissingData(X, column):
    from sklearn.preprocessing import Imputer
    imputer = Imputer(missing_values='NaN', strategy='median')
    X[:,column:column + 1] = imputer.fit_transform(X[:,column:column + 1])
    return X
//  [python]
def fillMissingData(X, column):
    from sklearn.impute import SimpleImputer
    imputer = SimpleImputer(missing_values=np.nan, strategy='median')
    X[:,column:column + 1] = imputer.fit_transform(X[:,column:column + 1])
    return X
//  [python]
def test_calibration_nan_imputer(ensemble):
    """Test that calibration can accept nan"""
    X, y = make_classification(n_samples=10, n_features=2,
                               n_informative=2, n_redundant=0,
                               random_state=42)
    X[0, 0] = np.nan
    clf = Pipeline(
        [('imputer', Imputer()),
         ('rf', RandomForestClassifier(n_estimators=1))])
    clf_c = CalibratedClassifierCV(
        clf, cv=2, method='isotonic', ensemble=ensemble
    )
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def test_calibration_nan_imputer(ensemble):
    """Test that calibration can accept nan"""
    X, y = make_classification(n_samples=10, n_features=2,
                               n_informative=2, n_redundant=0,
                               random_state=42)
    X[0, 0] = np.nan
    clf = Pipeline(
        [('imputer', SimpleImputer()),
         ('rf', RandomForestClassifier(n_estimators=1))])
    clf_c = CalibratedClassifierCV(
        clf, cv=2, method='isotonic', ensemble=ensemble
    )
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def pcaPreprocess(data, doPCA=False, n_components=-1, missing=False,
                  missing_strategy='median', random_state=1234):
    r"""Preprocess data using PCA.

    Parameters
    ==========
    data : array of shape (n_individuals, n_dimensions)
        Data matrix.
    doPCA : bool, optional (default = False)
        Apply PCA pre-processing.
    n_components : int, optional (default = -1)
        Number of components for PCA pre-processing.
        If set to -1, keep principal components
        accounting for 80% of data variance.
    missing : bool, optional (default = True)
        Replace missing values (calls scikit-learn functions).
    missing_strategy : str (default = 'median')
        Scikit-learn missing data strategy.
    random_state : int (default = 1234)
        Random state.

    Returns
    =======
    array of shape (n_individuals, n_components)
        Data projected onto principal axes.
    """
    if missing:
        imp = Imputer(strategy=missing_strategy)
        data = imp.fit_transform(data)
    scaler = StandardScaler()
    data = scaler.fit_transform(data)
    if n_components == -1:
        n_components = 0.80
    if doPCA:
        pca = PCA(random_state=random_state, n_components=n_components)
        data = pca.fit_transform(data)
        n_components = pca.n_components_
        print("Used %s components explaining %s%% of the variance\n" %
              (n_components,
               pca.explained_variance_ratio_.cumsum()[n_components-1]*100))
    return(data)
//  [python]
def pcaPreprocess(data, doPCA=False, n_components=-1, missing=False,
                  missing_strategy='median', random_state=1234):
    r"""Preprocess data using PCA.

    Parameters
    ==========
    data : array of shape (n_individuals, n_dimensions)
        Data matrix.
    doPCA : bool, optional (default = False)
        Apply PCA pre-processing.
    n_components : int, optional (default = -1)
        Number of components for PCA pre-processing.
        If set to -1, keep principal components
        accounting for 80% of data variance.
    missing : bool, optional (default = True)
        Replace missing values (calls scikit-learn functions).
    missing_strategy : str (default = 'median')
        Scikit-learn missing data strategy.
    random_state : int (default = 1234)
        Random state.

    Returns
    =======
    array of shape (n_individuals, n_components)
        Data projected onto principal axes.
    """
    if missing:
        imp = SimpleImputer(strategy=missing_strategy)
        data = imp.fit_transform(data)
    scaler = StandardScaler()
    data = scaler.fit_transform(data)
    if n_components == -1:
        n_components = 0.80
    if doPCA:
        pca = PCA(random_state=random_state, n_components=n_components)
        data = pca.fit_transform(data)
        n_components = pca.n_components_
        print("Used %s components explaining %s%% of the variance\n" %
              (n_components,
               pca.explained_variance_ratio_.cumsum()[n_components-1]*100))
    return(data)
//  [python]
def processTrainTest(train, test, doPCA, n_components, missing=False,
                     missing_strategy='median', random_state=1234):
    r"""Preprocess train and test data using PCA.

    Parameters
    ==========
    train : array of shape (n_individuals, n_train)
        Train data matrix.
    test : array of shape (n_individuals, n_test)
        Test data matrix.
    doPCA : bool, optional (default = False)
        Apply PCA pre-processing.
    n_components : int, optional (default = -1)
        Number of components for PCA pre-processing.
        If set to -1, keep principal components
        accounting for 80% of data variance.
    missing : bool, optional (default = True)
        Replace missing values (calls scikit-learn functions).
    missing_strategy : str (default = 'median')
        Scikit-learn missing data strategy.
    random_state : int (default = 1234)
        Random state.

    Returns
    =======
    instance of :class:`~ugtm.ugtm_preprocess.ProcessedTrainTest`
    """
    if missing:
        imp = Imputer(strategy=missing_strategy)
        train = imp.fit_transform(train)
        test = imp.transform(test)
    scaler = StandardScaler()
    train = scaler.fit_transform(train)
    test = scaler.transform(test)
    if(n_components == -1):
        n_components = 0.80
    if doPCA:
        pca = PCA(random_state=random_state, n_components=n_components)
        train = pca.fit_transform(train)
        test = pca.transform(test)
    return(ProcessedTrainTest(train, test))
//  [python]
def processTrainTest(train, test, doPCA, n_components, missing=False,
                     missing_strategy='median', random_state=1234):
    r"""Preprocess train and test data using PCA.

    Parameters
    ==========
    train : array of shape (n_individuals, n_train)
        Train data matrix.
    test : array of shape (n_individuals, n_test)
        Test data matrix.
    doPCA : bool, optional (default = False)
        Apply PCA pre-processing.
    n_components : int, optional (default = -1)
        Number of components for PCA pre-processing.
        If set to -1, keep principal components
        accounting for 80% of data variance.
    missing : bool, optional (default = True)
        Replace missing values (calls scikit-learn functions).
    missing_strategy : str (default = 'median')
        Scikit-learn missing data strategy.
    random_state : int (default = 1234)
        Random state.

    Returns
    =======
    instance of :class:`~ugtm.ugtm_preprocess.ProcessedTrainTest`
    """
    if missing:
        imp = SimpleImputer(strategy=missing_strategy)
        train = imp.fit_transform(train)
        test = imp.transform(test)
    scaler = StandardScaler()
    train = scaler.fit_transform(train)
    test = scaler.transform(test)
    if(n_components == -1):
        n_components = 0.80
    if doPCA:
        pca = PCA(random_state=random_state, n_components=n_components)
        train = pca.fit_transform(train)
        test = pca.transform(test)
    return(ProcessedTrainTest(train, test))
//  [python]
set_config(transform_output="pandas")

num_pipe = make_pipeline(Imputer(), StandardScaler())
//  [python]
set_config(transform_output="pandas")

num_pipe = make_pipeline(SimpleImputer(), StandardScaler())
//  [python]
def impute_numeric(self, columns, strategy='median'):
        imputer = Imputer(missing_values = np.nan, strategy = strategy)
        self.data[columns] = imputer.fit_transform(self.data[columns])
//  [python]
def impute_numeric(self, columns, strategy='median'):
        imputer = SimpleImputer(missing_values = np.nan, strategy = strategy)
        self.data[columns] = imputer.fit_transform(self.data[columns])
//  [python]
def preprocess(raw_data_s3_path: str) -> tuple[pd.DataFrame, ...]:
    df = pd.read_csv(
        raw_data_s3_path,
        header=None,
        names=feature_columns_names + [label_column],
        dtype=merge_two_dicts(feature_columns_dtype, label_column_dtype),
    )
    numeric_features = list(feature_columns_names)
    numeric_features.remove("sex")
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = ["sex"]
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocess = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    y = df.pop("rings")
    X_pre = preprocess.fit_transform(df)
    y_pre = y.to_numpy().reshape(len(y), 1)

    X = np.concatenate((y_pre, X_pre), axis=1)

    np.random.shuffle(X)
    train, validation, test = np.split(X, [int(0.7 * len(X)), int(0.85 * len(X))])
    return pd.DataFrame(train), pd.DataFrame(validation), pd.DataFrame(test)
//  [python]
def preprocess(raw_data_s3_path: str) -> tuple[pd.DataFrame, ...]:
    df = pd.read_csv(
        raw_data_s3_path,
        header=None,
        names=feature_columns_names + [label_column],
        dtype=merge_two_dicts(feature_columns_dtype, label_column_dtype),
    )
    numeric_features = list(feature_columns_names)
    numeric_features.remove("sex")
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = ["sex"]
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocess = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    y = df.pop("rings")
    X_pre = preprocess.fit_transform(df)
    y_pre = y.to_numpy().reshape(len(y), 1)

    X = np.concatenate((y_pre, X_pre), axis=1)

    np.random.shuffle(X)
    train, validation, test = np.split(X, [int(0.7 * len(X)), int(0.85 * len(X))])
    return pd.DataFrame(train), pd.DataFrame(validation), pd.DataFrame(test)
//  [python]
def impute(df, config):
    #XXX This ensures that out scaling will have the same mean. We still need to check the variance
    if not hasattr(config, 'missing_data_label'):
        return df, None
    else:
        imp = Imputer(missing_values=config.missing_data_label, strategy='mean')
        mask = df.applymap(lambda x: True if x == config.missing_data_label else False)
        data = df.values
        col_mask = (data == config.missing_data_label).all(axis=0)
        data[:,~col_mask] = imp.fit_transform(data)
        return data, mask
//  [python]
def impute(df, config):
    #XXX This ensures that out scaling will have the same mean. We still need to check the variance
    if not hasattr(config, 'missing_data_label'):
        return df, None
    else:
        imp = SimpleImputer(missing_values=config.missing_data_label, strategy='mean')
        mask = df.applymap(lambda x: True if x == config.missing_data_label else False)
        data = df.values
        col_mask = (data == config.missing_data_label).all(axis=0)
        data[:,~col_mask] = imp.fit_transform(data)
        return data, mask
//  [python]
def test_calibration_nan_imputer(ensemble):
    """Test that calibration can accept nan"""
    X, y = make_classification(
        n_samples=10, n_features=2, n_informative=2, n_redundant=0, random_state=42
    )
    X[0, 0] = np.nan
    clf = Pipeline(
        [("imputer", Imputer()), ("rf", RandomForestClassifier(n_estimators=1))]
    )
    clf_c = CalibratedClassifierCV(clf, cv=2, method="isotonic", ensemble=ensemble)
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def test_calibration_nan_imputer(ensemble):
    """Test that calibration can accept nan"""
    X, y = make_classification(
        n_samples=10, n_features=2, n_informative=2, n_redundant=0, random_state=42
    )
    X[0, 0] = np.nan
    clf = Pipeline(
        [("imputer", SimpleImputer()), ("rf", RandomForestClassifier(n_estimators=1))]
    )
    clf_c = CalibratedClassifierCV(clf, cv=2, method="isotonic", ensemble=ensemble)
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def test_model_info_pipeline(iris_adaboost):
    # Arrange
    simple_pipeline = Pipeline([('nan_handling', Imputer(strategy='most_frequent')),
                                ('adaboost', iris_adaboost)])
    # Act
    result = ModelInfo().run(simple_pipeline)
    # Assert
    assert_model_result(result)
//  [python]
def test_model_info_pipeline(iris_adaboost):
    # Arrange
    simple_pipeline = Pipeline([('nan_handling', SimpleImputer(strategy='most_frequent')),
                                ('adaboost', iris_adaboost)])
    # Act
    result = ModelInfo().run(simple_pipeline)
    # Assert
    assert_model_result(result)
//  [python]
def get_data_transformer_object(self)->ColumnTransformer:
        try:
            schema_file_path = self.data_validation_artifact.schema_file_path

            dataset_schema = read_yaml_file(file_path=schema_file_path)

            numerical_columns = dataset_schema[NUMERICAL_COLUMN_KEY]
            categorical_columns = dataset_schema[CATEGORICAL_COLUMN_KEY]


            num_pipeline = Pipeline(steps=[
                ('imputer', Imputer(strategy="median")),
                ('feature_generator', FeatureGenerator(
                    add_bedrooms_per_room=self.data_transformation_config.add_bedroom_per_room,
                    columns=numerical_columns
                )),
                ('scaler', StandardScaler())
            ]
            )

            cat_pipeline = Pipeline(steps=[
                 ('impute', Imputer(strategy="most_frequent")),
                 ('one_hot_encoder', OneHotEncoder()),
                 ('scaler', StandardScaler(with_mean=False))
            ]
            )

            logging.info(f"Categorical columns: {categorical_columns}")
            logging.info(f"Numerical columns: {numerical_columns}")


            preprocessing = ColumnTransformer([
                ('num_pipeline', num_pipeline, numerical_columns),
                ('cat_pipeline', cat_pipeline, categorical_columns),
            ])
            return preprocessing

        except Exception as e:
            raise HousingException(e,sys) from e
//  [python]
def get_data_transformer_object(self)->ColumnTransformer:
        try:
            schema_file_path = self.data_validation_artifact.schema_file_path

            dataset_schema = read_yaml_file(file_path=schema_file_path)

            numerical_columns = dataset_schema[NUMERICAL_COLUMN_KEY]
            categorical_columns = dataset_schema[CATEGORICAL_COLUMN_KEY]


            num_pipeline = Pipeline(steps=[
                ('imputer', SimpleImputer(strategy="median")),
                ('feature_generator', FeatureGenerator(
                    add_bedrooms_per_room=self.data_transformation_config.add_bedroom_per_room,
                    columns=numerical_columns
                )),
                ('scaler', StandardScaler())
            ]
            )

            cat_pipeline = Pipeline(steps=[
                 ('impute', SimpleImputer(strategy="most_frequent")),
                 ('one_hot_encoder', OneHotEncoder()),
                 ('scaler', StandardScaler(with_mean=False))
            ]
            )

            logging.info(f"Categorical columns: {categorical_columns}")
            logging.info(f"Numerical columns: {numerical_columns}")


            preprocessing = ColumnTransformer([
                ('num_pipeline', num_pipeline, numerical_columns),
                ('cat_pipeline', cat_pipeline, categorical_columns),
            ])
            return preprocessing

        except Exception as e:
            raise HousingException(e,sys) from e
//  [python]
def __init__(self, PredictionModel, sensitiveAttribute, metadata, quids=None):
        """
        Parent class for simple regression attribute inference attack

        :param PredictionModel: object: sklearn-type prediction model
        :param sensitiveAttribute: string: name of a column in a DataFrame that is considered the unknown, sensitive attribute
        :param metadata: dict: schema for the data to be attacked
        :param backgroundKnowledge: pd.DataFrame: adversary's background knowledge dataset
        """

        self.PredictionModel = PredictionModel
        self.sensitiveAttribute = sensitiveAttribute

        self.metadata, self.knownAttributes, self.categoricalAttributes, self.nfeatures = self._read_meta(metadata, quids)

        self.ImputerCat = Imputer(strategy='most_frequent')
        self.ImputerNum = Imputer(strategy='median')

        self.trained = False

        self.__name__ = f'{self.PredictionModel.__class__.__name__}'
//  [python]
def __init__(self, PredictionModel, sensitiveAttribute, metadata, quids=None):
        """
        Parent class for simple regression attribute inference attack

        :param PredictionModel: object: sklearn-type prediction model
        :param sensitiveAttribute: string: name of a column in a DataFrame that is considered the unknown, sensitive attribute
        :param metadata: dict: schema for the data to be attacked
        :param backgroundKnowledge: pd.DataFrame: adversary's background knowledge dataset
        """

        self.PredictionModel = PredictionModel
        self.sensitiveAttribute = sensitiveAttribute

        self.metadata, self.knownAttributes, self.categoricalAttributes, self.nfeatures = self._read_meta(metadata, quids)

        self.ImputerCat = SimpleImputer(strategy='most_frequent')
        self.ImputerNum = SimpleImputer(strategy='median')

        self.trained = False

        self.__name__ = f'{self.PredictionModel.__class__.__name__}'
//  [python]
y = data["survived"]

# SimpleImputer on string is not available
# for string in ONNX-ML specifications.
# So we do it beforehand.
for cat in ["embarked", "sex", "pclass"]:
    X[cat].fillna("missing", inplace=True)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

numeric_features = ["age", "fare"]
numeric_transformer = Pipeline(
    steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
y = data["survived"]

# SimpleImputer on string is not available
# for string in ONNX-ML specifications.
# So we do it beforehand.
for cat in ["embarked", "sex", "pclass"]:
    X[cat].fillna("missing", inplace=True)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

numeric_features = ["age", "fare"]
numeric_transformer = Pipeline(
    steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
def preprocess_static_feature(
    static_feature_dict,
    imputation_strategy="median",
    standardize=False
):
  """Preprocessing for a dictionary of static features.

  Args:
    static_feature_dict: Dictionary of float values.
    imputation_strategy: "median" or "mean" or "most_frequent" imputation.
    standardize: If true, apply MinMax scaling.

  Returns:
    Preprocessing static features array and the fitted scaler.
  """
  if imputation_strategy not in ["mean", "median", "most_frequent", "constant"]:
    raise ValueError(
        "Given imputation strategy {} is not supported. "
        "Use value from ['mean', 'median', 'most_frequent', 'constant'] "
        "as an imputation strategy.".format(imputation_strategy))

  location_keys = list(static_feature_dict.keys())

  static_values = np.empty([len(location_keys), 1])
  for idx, location_key in enumerate(location_keys):
    if static_feature_dict[location_key] is not None:
      static_values[idx] = static_feature_dict[location_key]
    else:
      static_values[idx] = np.nan

  if imputation_strategy:
    # Check whether all elements are NaNs.
    if np.all(np.isnan(static_values)):
      static_values = np.zeros_like(static_values)
    else:
      imputer = Imputer(
          missing_values=np.nan, strategy=imputation_strategy)
      static_values = imputer.fit_transform(static_values)

  scaler = None
  if standardize:
    # Check if there are at least two elements with different values to avoid
    # issues.
    if np.unique(static_values).size < 2:
      static_values = np.zeros_like(static_values)
    else:
      scaler = preprocessing.MinMaxScaler()
      static_values = scaler.fit_transform(static_values)

  static_feature_dict = {}
  for ind, location_key in enumerate(location_keys):
    static_feature_dict[location_key] = static_values[ind, 0]

  return static_feature_dict, scaler
//  [python]
def preprocess_static_feature(
    static_feature_dict,
    imputation_strategy="median",
    standardize=False
):
  """Preprocessing for a dictionary of static features.

  Args:
    static_feature_dict: Dictionary of float values.
    imputation_strategy: "median" or "mean" or "most_frequent" imputation.
    standardize: If true, apply MinMax scaling.

  Returns:
    Preprocessing static features array and the fitted scaler.
  """
  if imputation_strategy not in ["mean", "median", "most_frequent", "constant"]:
    raise ValueError(
        "Given imputation strategy {} is not supported. "
        "Use value from ['mean', 'median', 'most_frequent', 'constant'] "
        "as an imputation strategy.".format(imputation_strategy))

  location_keys = list(static_feature_dict.keys())

  static_values = np.empty([len(location_keys), 1])
  for idx, location_key in enumerate(location_keys):
    if static_feature_dict[location_key] is not None:
      static_values[idx] = static_feature_dict[location_key]
    else:
      static_values[idx] = np.nan

  if imputation_strategy:
    # Check whether all elements are NaNs.
    if np.all(np.isnan(static_values)):
      static_values = np.zeros_like(static_values)
    else:
      imputer = SimpleImputer(
          missing_values=np.nan, strategy=imputation_strategy)
      static_values = imputer.fit_transform(static_values)

  scaler = None
  if standardize:
    # Check if there are at least two elements with different values to avoid
    # issues.
    if np.unique(static_values).size < 2:
      static_values = np.zeros_like(static_values)
    else:
      scaler = preprocessing.MinMaxScaler()
      static_values = scaler.fit_transform(static_values)

  static_feature_dict = {}
  for ind, location_key in enumerate(location_keys):
    static_feature_dict[location_key] = static_values[ind, 0]

  return static_feature_dict, scaler
//  [python]
def get_pipeline(train) -> dm.Pipeline:
    imputer = Imputer(strategy='median')
    scaler = StandardScaler()

    numeric_transformer = Pipeline(steps=[
        ('imputer', imputer),
        ('scaler', scaler)])

    cat_imputer = Imputer(strategy='constant', fill_value='missing')
    cat_onehot = OneHotEncoder(handle_unknown='ignore')

    categorical_transformer = Pipeline(steps=[
        ('imputer', cat_imputer),
        ('onehot', cat_onehot)])
    numeric_features = train.select_dtypes(include=['int64', 'float64']).columns
    categorical_features = train.select_dtypes(include=['object']).columns
    from sklearn.compose import ColumnTransformer
    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categorical_features)])

    classifiers = [
        RandomForestClassifier(),
        GradientBoostingClassifier()
    ]
    pipeline = dm.Pipeline()
    node_pre = dm.EstimatorNode('preprocess', preprocessor)
    node_rf = dm.EstimatorNode('random_forest', classifiers[0])
    node_gb = dm.EstimatorNode('gradient_boost', classifiers[1])

    pipeline.add_edge(node_pre, node_rf)
    pipeline.add_edge(node_pre, node_gb)

    return pipeline
//  [python]
def get_pipeline(train) -> dm.Pipeline:
    imputer = SimpleImputer(strategy='median')
    scaler = StandardScaler()

    numeric_transformer = Pipeline(steps=[
        ('imputer', imputer),
        ('scaler', scaler)])

    cat_imputer = SimpleImputer(strategy='constant', fill_value='missing')
    cat_onehot = OneHotEncoder(handle_unknown='ignore')

    categorical_transformer = Pipeline(steps=[
        ('imputer', cat_imputer),
        ('onehot', cat_onehot)])
    numeric_features = train.select_dtypes(include=['int64', 'float64']).columns
    categorical_features = train.select_dtypes(include=['object']).columns
    from sklearn.compose import ColumnTransformer
    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categorical_features)])

    classifiers = [
        RandomForestClassifier(),
        GradientBoostingClassifier()
    ]
    pipeline = dm.Pipeline()
    node_pre = dm.EstimatorNode('preprocess', preprocessor)
    node_rf = dm.EstimatorNode('random_forest', classifiers[0])
    node_gb = dm.EstimatorNode('gradient_boost', classifiers[1])

    pipeline.add_edge(node_pre, node_rf)
    pipeline.add_edge(node_pre, node_gb)

    return pipeline
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        dataset = load_concrete(return_dataset=True)
        X, y = dataset.to_data()
        features = dataset.meta["features"]
        features = list(map(lambda s: s.title(), features))

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('fi', FeatureImportances(Lasso(random_state=42), labels=features, relative=False))
        ])

        model.fit(X, y)
        model['fi'].finalize()
        self.assert_images_similar(model['fi'], tol=17.5)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        dataset = load_concrete(return_dataset=True)
        X, y = dataset.to_data()
        features = dataset.meta["features"]
        features = list(map(lambda s: s.title(), features))

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('fi', FeatureImportances(Lasso(random_state=42), labels=features, relative=False))
        ])

        model.fit(X, y)
        model['fi'].finalize()
        self.assert_images_similar(model['fi'], tol=17.5)
//  [python]
def fillMissingData(X, inicioColuna, fimColuna):
    from sklearn.preprocessing import Imputer
    imputer = Imputer(missing_values='NaN', strategy='median')
    X[:,inicioColuna:fimColuna + 1] = imputer.fit_transform(X[:,inicioColuna:fimColuna + 1])
    return X
//  [python]
def fillMissingData(X, inicioColuna, fimColuna):
    from sklearn.impute import SimpleImputer
    imputer = SimpleImputer(missing_values=np.nan, strategy='median')
    X[:,inicioColuna:fimColuna + 1] = imputer.fit_transform(X[:,inicioColuna:fimColuna + 1])
    return X
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        dataset = load_concrete(return_dataset=True)
        X, y = dataset.to_data()
        features = dataset.meta["features"]
        features = list(map(lambda s: s.title(), features))

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('fi', feature_importances(Lasso(random_state=42), X, y, labels=features, relative=False, show=False))
        ])
        self.assert_images_similar(model['fi'], tol=17.5)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        dataset = load_concrete(return_dataset=True)
        X, y = dataset.to_data()
        features = dataset.meta["features"]
        features = list(map(lambda s: s.title(), features))

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('fi', feature_importances(Lasso(random_state=42), X, y, labels=features, relative=False, show=False))
        ])
        self.assert_images_similar(model['fi'], tol=17.5)