jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def column_processor(self):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        from sklearn.preprocessing import Imputer
        
        category_cols, numerical_cols = self.num_cat_variables()
        category_map = self.gen_category_map(self.data, category_cols)

        
        ordinal_features = [x for x in range(len(self.data.columns)) if x not in list(category_map.keys())]
        ordinal_transformer = Pipeline(steps=[('imputer', Imputer(strategy='median')),
                                    ('scaler', StandardScaler())])
        
        categorical_features = list(category_map.keys())
        categorical_transformer = Pipeline(steps=[('imputer', Imputer(strategy='most_frequent')),
                                        ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
        
        #pass the pipeline into the transformer
        preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                            ('cat', categorical_transformer, categorical_features)])
        #preprocessor.fit(data)
        return preprocessor
//  [python]
def column_processor(self):
        '''
        Input:
        data : dataframe or x_test
        Column Transformer: Sklearn ColumnTransformer that applies transformers to columns of an array or pandas DataFrame.

        Returns:
        Column Transformer function after fit. 
        '''
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
        from sklearn.compose import ColumnTransformer
        
        category_cols, numerical_cols = self.num_cat_variables()
        category_map = self.gen_category_map(self.data, category_cols)

        
        ordinal_features = [x for x in range(len(self.data.columns)) if x not in list(category_map.keys())]
        ordinal_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),
                                    ('scaler', StandardScaler())])
        
        categorical_features = list(category_map.keys())
        categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')),
                                        ('onehot', OneHotEncoder(drop='first', handle_unknown='error'))])
        
        #pass the pipeline into the transformer
        preprocessor = ColumnTransformer(transformers=[('num', ordinal_transformer, ordinal_features),
                                            ('cat', categorical_transformer, categorical_features)])
        #preprocessor.fit(data)
        return preprocessor
//  [python]
def _get_estimator(pblm, clf_key):
        """
        Returns sklearn classifier
        """
        tup = clf_key.split('-')
        wrap_type = None if len(tup) == 1 else tup[1]
        est_type = tup[0]
        multiclass_wrapper = {
            None: ut.identity,
            'OVR': sklearn.multiclass.OneVsRestClassifier,
            'OVO': sklearn.multiclass.OneVsOneClassifier,
        }[wrap_type]
        est_class = {
            'RF': sklearn.ensemble.RandomForestClassifier,
            'SVC': sklearn.svm.SVC,
            'Logit': sklearn.linear_model.LogisticRegression,
            'MLP': sklearn.neural_network.MLPClassifier,
        }[est_type]

        est_kw1, est_kw2 = pblm._estimator_params(est_type)
        est_params = ut.merge_dicts(est_kw1, est_kw2)

        # steps = []
        # steps.append((est_type, est_class(**est_params)))
        # if wrap_type is not None:
        #     steps.append((wrap_type, multiclass_wrapper))
        if est_type == 'MLP':

            def clf_partial():
                pipe = sklearn.pipeline.Pipeline(
                    [
                        ('inputer', sklearn.preprocessing.Imputer(strategy='mean')),
                        # ('scale', sklearn.preprocessing.StandardScaler),
                        ('est', est_class(**est_params)),
                    ]
                )
                return multiclass_wrapper(pipe)

        elif est_type == 'Logit':

            def clf_partial():
                pipe = sklearn.pipeline.Pipeline(
                    [
                        ('inputer', sklearn.preprocessing.Imputer(strategy='mean')),
                        ('est', est_class(**est_params)),
                    ]
                )
                return multiclass_wrapper(pipe)

        else:

            def clf_partial():
                return multiclass_wrapper(est_class(**est_params))

        return clf_partial
//  [python]
def _get_estimator(pblm, clf_key):
        """
        Returns sklearn classifier
        """
        tup = clf_key.split('-')
        wrap_type = None if len(tup) == 1 else tup[1]
        est_type = tup[0]
        multiclass_wrapper = {
            None: ut.identity,
            'OVR': sklearn.multiclass.OneVsRestClassifier,
            'OVO': sklearn.multiclass.OneVsOneClassifier,
        }[wrap_type]
        est_class = {
            'RF': sklearn.ensemble.RandomForestClassifier,
            'SVC': sklearn.svm.SVC,
            'Logit': sklearn.linear_model.LogisticRegression,
            'MLP': sklearn.neural_network.MLPClassifier,
        }[est_type]

        est_kw1, est_kw2 = pblm._estimator_params(est_type)
        est_params = ut.merge_dicts(est_kw1, est_kw2)

        # steps = []
        # steps.append((est_type, est_class(**est_params)))
        # if wrap_type is not None:
        #     steps.append((wrap_type, multiclass_wrapper))
        if est_type == 'MLP':

            def clf_partial():
                pipe = sklearn.pipeline.Pipeline(
                    [
                        ('inputer', sklearn.impute.SimpleImputer(strategy='mean')),
                        # ('scale', sklearn.preprocessing.StandardScaler),
                        ('est', est_class(**est_params)),
                    ]
                )
                return multiclass_wrapper(pipe)

        elif est_type == 'Logit':

            def clf_partial():
                pipe = sklearn.pipeline.Pipeline(
                    [
                        ('inputer', sklearn.impute.SimpleImputer(strategy='mean')),
                        ('est', est_class(**est_params)),
                    ]
                )
                return multiclass_wrapper(pipe)

        else:

            def clf_partial():
                return multiclass_wrapper(est_class(**est_params))

        return clf_partial
//  [python]
def test_fit_constant(data):
    a = s.Imputer(strategy="constant", fill_value=-999.0)
    b = dask_ml.impute.SimpleImputer(strategy="constant", fill_value=-999.0)

    expected = a.fit_transform(X)
    result = b.fit_transform(data)

    assert_estimator_equal(a, b, exclude=["statistics_"])
    np.testing.assert_array_almost_equal(a.statistics_, np.asarray(b.statistics_))
    assert isinstance(result, type(data))
    if isinstance(data, (pd.DataFrame, dd.DataFrame)):
        result = result.values

    da.utils.assert_eq(result, expected)
//  [python]
def test_fit_constant(data):
    a = sklearn.impute.SimpleImputer(strategy="constant", fill_value=-999.0)
    b = dask_ml.impute.SimpleImputer(strategy="constant", fill_value=-999.0)

    expected = a.fit_transform(X)
    result = b.fit_transform(data)

    assert_estimator_equal(a, b, exclude=["statistics_"])
    np.testing.assert_array_almost_equal(a.statistics_, np.asarray(b.statistics_))
    assert isinstance(result, type(data))
    if isinstance(data, (pd.DataFrame, dd.DataFrame)):
        result = result.values

    da.utils.assert_eq(result, expected)
//  [python]
def test_transform(data):
    a = sklearn.preprocessing.Imputer()
    b = dask_ml.impute.SimpleImputer()

    expected = a.fit_transform(X)
    result = b.fit_transform(data)

    assert isinstance(result, type(data))
    if isinstance(data, (pd.DataFrame, dd.DataFrame)):
        result = result.values

    da.utils.assert_eq(result, expected)
//  [python]
def test_transform(data):
    a = sklearn.impute.SimpleImputer()
    b = dask_ml.impute.SimpleImputer()

    expected = a.fit_transform(X)
    result = b.fit_transform(data)

    assert isinstance(result, type(data))
    if isinstance(data, (pd.DataFrame, dd.DataFrame)):
        result = result.values

    da.utils.assert_eq(result, expected)
//  [python]
def test_sklearn_impute(strategy, pd_X, benchmark):
    imputer = Imputer(strategy=strategy)
    benchmark(pd_gb_transform, pd_X, imputer)
//  [python]
def test_sklearn_impute(strategy, pd_X, benchmark):
    imputer = SimpleImputer(strategy=strategy)
    benchmark(pd_gb_transform, pd_X, imputer)
//  [python]
def my_imputer(method = 'MEAN'):
    if method == 'MEAN':
        imputer = Imputer(missing_values = np.nan, strategy = 'mean')
    elif method == 'MEDIAN':
        imputer = Imputer(missing_values = np.nan, strategy = 'median')
    elif method == 'KNN':
        imputer = KNNImputer(n_neighbors = 10, weights = 'uniform')
    elif method == 'MICE':
        imputer = IterativeImputer(max_iter = 10, random_state = 2020)
    return imputer
//  [python]
def my_imputer(method = 'MEAN'):
    if method == 'MEAN':
        imputer = SimpleImputer(missing_values = np.nan, strategy = 'mean')
    elif method == 'MEDIAN':
        imputer = SimpleImputer(missing_values = np.nan, strategy = 'median')
    elif method == 'KNN':
        imputer = KNNImputer(n_neighbors = 10, weights = 'uniform')
    elif method == 'MICE':
        imputer = IterativeImputer(max_iter = 10, random_state = 2020)
    return imputer
//  [python]
def sparse_data_transformed():
    tests_dir = __file__
    os.chdir(os.path.dirname(tests_dir))

    decoder = arff.ArffDecoder()
    with open(os.path.join("datasets", "dataset.arff")) as fh:
        dataset = decoder.decode(fh, encode_nominal=True)

    # -1 because the last attribute is the class
    attribute_types = [
        "numeric" if type(type_) != list else "nominal"
        for name, type_ in dataset["attributes"][:-1]
    ]
    feat_type = {
        i: "categorical" if attribute == "nominal" else "numerical"
        for i, attribute in enumerate(attribute_types)
    }

    data = np.array(dataset["data"], dtype=np.float64)
    X = data[:, :-1]
    y = data[:, -1].reshape((-1,))

    # First, swap NaNs and zeros, because when converting an encoded
    # dense matrix to sparse, the values which are encoded to zero are lost
    X_sparse = X.copy()
    NaNs = ~np.isfinite(X_sparse)
    X_sparse[NaNs] = 0
    X_sparse = sparse.csr_matrix(X_sparse)

    ohe = FeatTypeSplit(feat_type=feat_type)
    X_transformed = X_sparse.copy()
    X_transformed = ohe.fit_transform(X_transformed)
    imp = Imputer(copy=False)
    X_transformed = imp.fit_transform(X_transformed)
    standard_scaler = StandardScaler(with_mean=False)
    X_transformed = standard_scaler.fit_transform(X_transformed)

    # Transform the array which indicates the numerical metafeatures
    feat_type_transformed = {i: "numerical" for i in range(X_transformed.shape[1])}

    X = X_sparse
    X_transformed = X_transformed
    y = y
    mf = meta_features.metafeatures
    helpers = meta_features.helper_functions
    logger = logging.getLogger()

    # Precompute some helper functions
    helpers.set_value(
        "PCA",
        helpers["PCA"](X_transformed, y, logger),
    )
    helpers.set_value(
        "MissingValues",
        helpers["MissingValues"](X, y, logger, feat_type),
    )
    mf.set_value(
        "NumberOfMissingValues",
        mf["NumberOfMissingValues"](X, y, logger, feat_type),
    )
    helpers.set_value(
        "NumSymbols",
        helpers["NumSymbols"](X, y, logger, feat_type),
    )
    helpers.set_value(
        "ClassOccurences",
        helpers["ClassOccurences"](X, y, logger),
    )
    helpers.set_value(
        "Skewnesses",
        helpers["Skewnesses"](X_transformed, y, logger, feat_type_transformed),
    )
    helpers.set_value(
        "Kurtosisses",
        helpers["Kurtosisses"](X_transformed, y, logger, feat_type_transformed),
    )
    return X_transformed, y, feat_type_transformed
//  [python]
def sparse_data_transformed():
    tests_dir = __file__
    os.chdir(os.path.dirname(tests_dir))

    decoder = arff.ArffDecoder()
    with open(os.path.join("datasets", "dataset.arff")) as fh:
        dataset = decoder.decode(fh, encode_nominal=True)

    # -1 because the last attribute is the class
    attribute_types = [
        "numeric" if type(type_) != list else "nominal"
        for name, type_ in dataset["attributes"][:-1]
    ]
    feat_type = {
        i: "categorical" if attribute == "nominal" else "numerical"
        for i, attribute in enumerate(attribute_types)
    }

    data = np.array(dataset["data"], dtype=np.float64)
    X = data[:, :-1]
    y = data[:, -1].reshape((-1,))

    # First, swap NaNs and zeros, because when converting an encoded
    # dense matrix to sparse, the values which are encoded to zero are lost
    X_sparse = X.copy()
    NaNs = ~np.isfinite(X_sparse)
    X_sparse[NaNs] = 0
    X_sparse = sparse.csr_matrix(X_sparse)

    ohe = FeatTypeSplit(feat_type=feat_type)
    X_transformed = X_sparse.copy()
    X_transformed = ohe.fit_transform(X_transformed)
    imp = SimpleImputer(copy=False)
    X_transformed = imp.fit_transform(X_transformed)
    standard_scaler = StandardScaler(with_mean=False)
    X_transformed = standard_scaler.fit_transform(X_transformed)

    # Transform the array which indicates the numerical metafeatures
    feat_type_transformed = {i: "numerical" for i in range(X_transformed.shape[1])}

    X = X_sparse
    X_transformed = X_transformed
    y = y
    mf = meta_features.metafeatures
    helpers = meta_features.helper_functions
    logger = logging.getLogger()

    # Precompute some helper functions
    helpers.set_value(
        "PCA",
        helpers["PCA"](X_transformed, y, logger),
    )
    helpers.set_value(
        "MissingValues",
        helpers["MissingValues"](X, y, logger, feat_type),
    )
    mf.set_value(
        "NumberOfMissingValues",
        mf["NumberOfMissingValues"](X, y, logger, feat_type),
    )
    helpers.set_value(
        "NumSymbols",
        helpers["NumSymbols"](X, y, logger, feat_type),
    )
    helpers.set_value(
        "ClassOccurences",
        helpers["ClassOccurences"](X, y, logger),
    )
    helpers.set_value(
        "Skewnesses",
        helpers["Skewnesses"](X_transformed, y, logger, feat_type_transformed),
    )
    helpers.set_value(
        "Kurtosisses",
        helpers["Kurtosisses"](X_transformed, y, logger, feat_type_transformed),
    )
    return X_transformed, y, feat_type_transformed
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived
        self.john = pd.DataFrame({'gender': ['male'], 'age': [25], 'class': ['1st'], 'embarked': ['Southampton'],
                                   'fare': [72], 'sibsp': [0], 'parch': 0}, index = ['John'])

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPClassifier(hidden_layer_sizes=(20, 20),
                                                           max_iter=400, random_state=0))])
        clf2 = Pipeline(steps=[('preprocessor', preprocessor),
                               ('classifier', MLPClassifier(hidden_layer_sizes=(50, 100, 50),
                                                            max_iter=400, random_state=0))])

        clf.fit(self.X, self.y)
        clf2.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, label="model1", verbose=False)
        self.exp2 = dx.Explainer(clf2, self.X, self.y, label="model2", verbose=False)

        # This plots should be supported
        self.reference_plots = [ROCContainer, ShapleyValuesContainer, BreakDownContainer, CeterisParibusContainer,
            FeatureImportanceContainer, PartialDependenceContainer, AccumulatedDependenceContainer, MetricsContainer, 
            FairnessCheckContainer, ShapleyValuesDependenceContainer, ShapleyValuesVariableImportanceContainer,
            VariableAgainstAnotherContainer, VariableDistributionContainer]
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived
        self.john = pd.DataFrame({'gender': ['male'], 'age': [25], 'class': ['1st'], 'embarked': ['Southampton'],
                                   'fare': [72], 'sibsp': [0], 'parch': 0}, index = ['John'])

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPClassifier(hidden_layer_sizes=(20, 20),
                                                           max_iter=400, random_state=0))])
        clf2 = Pipeline(steps=[('preprocessor', preprocessor),
                               ('classifier', MLPClassifier(hidden_layer_sizes=(50, 100, 50),
                                                            max_iter=400, random_state=0))])

        clf.fit(self.X, self.y)
        clf2.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, label="model1", verbose=False)
        self.exp2 = dx.Explainer(clf2, self.X, self.y, label="model2", verbose=False)

        # This plots should be supported
        self.reference_plots = [ROCContainer, ShapleyValuesContainer, BreakDownContainer, CeterisParibusContainer,
            FeatureImportanceContainer, PartialDependenceContainer, AccumulatedDependenceContainer, MetricsContainer, 
            FairnessCheckContainer, ShapleyValuesDependenceContainer, ShapleyValuesVariableImportanceContainer,
            VariableAgainstAnotherContainer, VariableDistributionContainer]
//  [python]
imp = Imputer(missing_values=np.nan, strategy='mean')

imp.fit([[1, 2], [np.nan, 3], [7, 6]])
//  [python]
imp = SimpleImputer(missing_values=np.nan, strategy='mean')

imp.fit([[1, 2], [np.nan, 3], [7, 6]])

SimpleImputer(add_indicator=False, copy=True, fill_value=None,
              missing_values=nan, strategy='mean', verbose=0)
//  [python]
def german_credit_raw_model(german_credit_raw_data):
    timer = Timer()

    columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"]

    numeric_transformer = Pipeline([("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())])

    columns_to_encode = [key for key in input_types.keys() if input_types[key] == "category"]

    categorical_transformer = Pipeline(
        [
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, columns_to_scale),
            ("cat", categorical_transformer, columns_to_encode),
        ]
    )
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=100, random_state=30))]
    )

    Y = german_credit_raw_data["default"]
    X = german_credit_raw_data.drop(columns="default")
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.20, random_state=30, stratify=Y  # NOSONAR
    )
    clf.fit(X_train, Y_train)

    model_score = clf.score(X_test, Y_test)
    timer.stop(f"Trained model with score: {model_score}")

    return clf
//  [python]
def german_credit_raw_model(german_credit_raw_data):
    timer = Timer()

    columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"]

    numeric_transformer = Pipeline([("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())])

    columns_to_encode = [key for key in input_types.keys() if input_types[key] == "category"]

    categorical_transformer = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, columns_to_scale),
            ("cat", categorical_transformer, columns_to_encode),
        ]
    )
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=100, random_state=30))]
    )

    Y = german_credit_raw_data["default"]
    X = german_credit_raw_data.drop(columns="default")
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.20, random_state=30, stratify=Y  # NOSONAR
    )
    clf.fit(X_train, Y_train)

    model_score = clf.score(X_test, Y_test)
    timer.stop(f"Trained model with score: {model_score}")

    return clf
//  [python]
def make_classifier_pipeline(X: pd.DataFrame, num_labels: int) -> Pipeline:
    """
    Make the classifier pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X: pd.DataFrame
        X containing all the required features for training
    num_labels: int
        The number of output labels

    Returns
    -------
    classifier_pipeline: Pipeline
        Classifier pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    # exclude any completely-missing columns when checking for numerics
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(steps=[("imputer", Imputer(strategy="mean"))])
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = KerasClassifier(
        build_fn=create_classification_model,
        num_features=len(num_features),
        num_labels=num_labels,
        epochs=30,
        batch_size=8,
        verbose=1,
    )

    # pipeline with preprocessor and estimator bundled
    classifier_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return classifier_pipeline
//  [python]
def make_classifier_pipeline(X: pd.DataFrame, num_labels: int) -> Pipeline:
    """
    Make the classifier pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X: pd.DataFrame
        X containing all the required features for training
    num_labels: int
        The number of output labels

    Returns
    -------
    classifier_pipeline: Pipeline
        Classifier pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    # exclude any completely-missing columns when checking for numerics
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = KerasClassifier(
        build_fn=create_classification_model,
        num_features=len(num_features),
        num_labels=num_labels,
        epochs=30,
        batch_size=8,
        verbose=1,
    )

    # pipeline with preprocessor and estimator bundled
    classifier_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return classifier_pipeline
//  [python]
def general_preprocessor():
    cat_transformer = Pipeline(
        steps=[('imputer_cat', Imputer(strategy='constant')), ('encoder', OrdinalEncoder())])
    num_transformer = Pipeline(steps=[('imputer_num', Imputer(strategy='mean')), ('scaler', StandardScaler())])

    preprocessor = DataFrameMapper(features=[(column_object_category_bool, cat_transformer),
                                             (column_number_exclude_timedelta, num_transformer)],
                                   input_df=True,
                                   df_out=True)
    return preprocessor
//  [python]
def general_preprocessor():
    cat_transformer = Pipeline(
        steps=[('imputer_cat', SimpleImputer(strategy='constant')), ('encoder', OrdinalEncoder())])
    num_transformer = Pipeline(steps=[('imputer_num', SimpleImputer(strategy='mean')), ('scaler', StandardScaler())])

    preprocessor = DataFrameMapper(features=[(column_object_category_bool, cat_transformer),
                                             (column_number_exclude_timedelta, num_transformer)],
                                   input_df=True,
                                   df_out=True)
    return preprocessor
//  [python]
def reset_pipeline(self):
        """Reset regressor pipeline."""
        steps = []
        numerical_features_idx = [
            int(np.where(self.features == tag)[0][0])
            for tag in self.numerical_features
        ]

        # Imputer
        if self._cfg['imputation_strategy'] != 'remove':
            imputer = Imputer(strategy=self._cfg['imputation_strategy'])
            steps.append(('imputer', imputer))

        # Scaler for numerical features
        if self._cfg['standardize_data']:
            x_scaler = ColumnTransformer(
                [('', StandardScaler(), numerical_features_idx)],
                remainder='passthrough',
            )
            steps.append(('x_scaler', x_scaler))

        # PCA for numerical features
        if self._cfg.get('pca'):
            pca = ColumnTransformer(
                [('', PCA(random_state=self.random_state),
                  numerical_features_idx)],
                remainder='passthrough',
            )
            steps.append(('pca', pca))

        # Final regressor
        final_parameters = self._load_final_parameters()
        final_regressor = self._CLF_TYPE(**final_parameters)

        # Transformer for labels if desired (if not, add pd to np converter)
        if self._cfg['standardize_data']:
            y_scaler = StandardScaler()
        else:
            y_scaler = StandardScaler(with_mean=False, with_std=False)
        transformed_target_regressor = AdvancedTransformedTargetRegressor(
            transformer=y_scaler, regressor=final_regressor)
        steps.append(('final', transformed_target_regressor))

        # Final pipeline
        if self._cfg['cache_intermediate_results']:
            if self._cfg['n_jobs'] is None or self._cfg['n_jobs'] == 1:
                memory = self._cfg['mlr_work_dir']
            else:
                logger.debug(
                    "Caching intermediate results of Pipeline is not "
                    "supported for multiple processes (using at most %i "
                    "processes)", self._cfg['n_jobs'])
                memory = None
        else:
            memory = None
        self._clf = AdvancedPipeline(steps, memory=memory)
        logger.info("Created pipeline with steps %s",
                    list(self._clf.named_steps.keys()))
//  [python]
def reset_pipeline(self):
        """Reset regressor pipeline."""
        steps = []
        numerical_features_idx = [
            int(np.where(self.features == tag)[0][0])
            for tag in self.numerical_features
        ]

        # Imputer
        if self._cfg['imputation_strategy'] != 'remove':
            imputer = SimpleImputer(strategy=self._cfg['imputation_strategy'])
            steps.append(('imputer', imputer))

        # Scaler for numerical features
        if self._cfg['standardize_data']:
            x_scaler = ColumnTransformer(
                [('', StandardScaler(), numerical_features_idx)],
                remainder='passthrough',
            )
            steps.append(('x_scaler', x_scaler))

        # PCA for numerical features
        if self._cfg.get('pca'):
            pca = ColumnTransformer(
                [('', PCA(random_state=self.random_state),
                  numerical_features_idx)],
                remainder='passthrough',
            )
            steps.append(('pca', pca))

        # Final regressor
        final_parameters = self._load_final_parameters()
        final_regressor = self._CLF_TYPE(**final_parameters)

        # Transformer for labels if desired (if not, add pd to np converter)
        if self._cfg['standardize_data']:
            y_scaler = StandardScaler()
        else:
            y_scaler = StandardScaler(with_mean=False, with_std=False)
        transformed_target_regressor = AdvancedTransformedTargetRegressor(
            transformer=y_scaler, regressor=final_regressor)
        steps.append(('final', transformed_target_regressor))

        # Final pipeline
        if self._cfg['cache_intermediate_results']:
            if self._cfg['n_jobs'] is None or self._cfg['n_jobs'] == 1:
                memory = self._cfg['mlr_work_dir']
            else:
                logger.debug(
                    "Caching intermediate results of Pipeline is not "
                    "supported for multiple processes (using at most %i "
                    "processes)", self._cfg['n_jobs'])
                memory = None
        else:
            memory = None
        self._clf = AdvancedPipeline(steps, memory=memory)
        logger.info("Created pipeline with steps %s",
                    list(self._clf.named_steps.keys()))
//  [python]
def make_regressor_pipeline(X: pd.DataFrame) -> Pipeline:
    """
    Make the regressor pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X: pd.DataFrame
        X containing all the required features for training

    Returns
    -------
    regressor_pipeline: Pipeline
        Regressor pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", Imputer(strategy="mean")), ("standardize", StandardScaler())]
    )
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = KerasRegressor(
        build_fn=create_regression_model,
        num_features=len(num_features),
        epochs=200,
        batch_size=8,
        verbose=1,
        validation_split=0.33,
        callbacks=[EarlyStopping(patience=20)],
    )

    # pipeline with preprocessor and estimator bundled
    regressor_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return regressor_pipeline
//  [python]
def make_regressor_pipeline(X: pd.DataFrame) -> Pipeline:
    """
    Make the regressor pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X: pd.DataFrame
        X containing all the required features for training

    Returns
    -------
    regressor_pipeline: Pipeline
        Regressor pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", SimpleImputer(strategy="mean")), ("standardize", StandardScaler())]
    )
    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = KerasRegressor(
        build_fn=create_regression_model,
        num_features=len(num_features),
        epochs=200,
        batch_size=8,
        verbose=1,
        validation_split=0.33,
        callbacks=[EarlyStopping(patience=20)],
    )

    # pipeline with preprocessor and estimator bundled
    regressor_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return regressor_pipeline
//  [python]
def _gen_ortho_learner_model_nuisance(self):
        return ModelNuisance(
            make_pipeline(Imputer(strategy='mean'), LinearRegression()),
            make_pipeline(Imputer(strategy='mean'), LinearRegression())
        )
//  [python]
def _gen_ortho_learner_model_nuisance(self):
        return ModelNuisance(
            make_pipeline(SimpleImputer(strategy='mean'), LinearRegression()),
            make_pipeline(SimpleImputer(strategy='mean'), LinearRegression())
        )
//  [python]
def _simple_impute(adata: AnnData, var_names: Iterable[str] | None, strategy: str) -> None:
    imputer = Imputer(strategy=strategy)
    if isinstance(var_names, Iterable):
        column_indices = _get_column_indices(adata, var_names)
        adata.X[::, column_indices] = imputer.fit_transform(adata.X[::, column_indices])
    else:
        adata.X = imputer.fit_transform(adata.X)
//  [python]
def _simple_impute(adata: AnnData, var_names: Iterable[str] | None, strategy: str) -> None:
    imputer = SimpleImputer(strategy=strategy)
    if isinstance(var_names, Iterable):
        column_indices = _get_column_indices(adata, var_names)
        adata.X[::, column_indices] = imputer.fit_transform(adata.X[::, column_indices])
    else:
        adata.X = imputer.fit_transform(adata.X)
//  [python]
def preprocess_random_forest(dataset, log):
    '''
    For random forest:
    - Do nothing for numerical features except null imputation. 
    - For categorical features, use ordinal encoding to map them into integers. 
    '''
    cat_columns, num_columns = [], []
    shift_amount = 0
    for i, f in enumerate(dataset.features):
        if f.is_target:
            shift_amount += 1
            continue
        elif f.is_categorical():
            cat_columns.append(i - shift_amount)
        else:
            num_columns.append(i - shift_amount)

    cat_pipeline = Pipeline([('imputer', Imputer(strategy='most_frequent')),
                             ('ordinal_encoder', OrdinalEncoder()),
                             ])
    
    num_pipeline = Pipeline([('imputer', Imputer(strategy='mean')),
                             ])
    
    data_pipeline = ColumnTransformer([
        ('categorical', cat_pipeline, cat_columns),
        ('numerical', num_pipeline, num_columns),
    ])

    data_pipeline.fit(np.concatenate([dataset.train.X, dataset.test.X], axis=0))
    
    X_train = data_pipeline.transform(dataset.train.X)
    X_test = data_pipeline.transform(dataset.test.X)  
    
    return X_train, X_test
//  [python]
def preprocess_random_forest(dataset, log):
    '''
    For random forest:
    - Do nothing for numerical features except null imputation. 
    - For categorical features, use ordinal encoding to map them into integers. 
    '''
    cat_columns, num_columns = [], []
    shift_amount = 0
    for i, f in enumerate(dataset.features):
        if f.is_target:
            shift_amount += 1
            continue
        elif f.is_categorical():
            cat_columns.append(i - shift_amount)
        else:
            num_columns.append(i - shift_amount)

    cat_pipeline = Pipeline([('imputer', SimpleImputer(strategy='most_frequent')),
                             ('ordinal_encoder', OrdinalEncoder()),
                             ])
    
    num_pipeline = Pipeline([('imputer', SimpleImputer(strategy='mean')),
                             ])
    
    data_pipeline = ColumnTransformer([
        ('categorical', cat_pipeline, cat_columns),
        ('numerical', num_pipeline, num_columns),
    ])

    data_pipeline.fit(np.concatenate([dataset.train.X, dataset.test.X], axis=0))
    
    X_train = data_pipeline.transform(dataset.train.X)
    X_test = data_pipeline.transform(dataset.test.X)  
    
    return X_train, X_test
//  [python]
def __init__(self, classifier, output_col, columns=[], prefixes=None, suffixes=None,
                 store_train_data=False,
                 store_transform_data=False,
                 scale_input=False, impute=False, raise_null=False,training_sampling_fn=None):
        self.classifier = classifier
        self.columns = columns
        self.prefixes = prefixes
        self.suffixes = suffixes
        self.output_col = output_col
        assert len(columns) > 0 or prefixes is not None
        self.scale_input = scale_input
        self.scaler = StandardScaler()
        self.impute = impute
        self.imp = Imputer(missing_values=np.nan, strategy='mean')
        self.imp_inf = Imputer(missing_values=np.inf, strategy='mean')
        self.raise_null = raise_null
        self.cols = None
        self.training_sampling_fn = training_sampling_fn
        self.train = None
        self.store_train_data = store_train_data
        self.store_transform_data = store_transform_data
        self.transform_data = None
//  [python]
def __init__(self, classifier, output_col, columns=[], prefixes=None, suffixes=None,
                 store_train_data=False,
                 store_transform_data=False,
                 scale_input=False, impute=False, raise_null=False,training_sampling_fn=None):
        self.classifier = classifier
        self.columns = columns
        self.prefixes = prefixes
        self.suffixes = suffixes
        self.output_col = output_col
        assert len(columns) > 0 or prefixes is not None
        self.scale_input = scale_input
        self.scaler = StandardScaler()
        self.impute = impute
        self.imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        self.imp_inf = SimpleImputer(missing_values=np.inf, strategy='mean')
        self.raise_null = raise_null
        self.cols = None
        self.training_sampling_fn = training_sampling_fn
        self.train = None
        self.store_train_data = store_train_data
        self.store_transform_data = store_transform_data
        self.transform_data = None
//  [python]
def test_missing(self):
        # create data with missing values
        np.random.seed(123)
        X = np.random.normal(size=(1000, 1))
        W = np.random.normal(size=(1000, 5))
        T = X[:, 0] + np.random.normal(size=(1000,))
        y = (1 + 0.5 * X[:, 0]) * T + X[:, 0] + np.random.normal(size=(1000,))
        mask = np.random.rand(*W.shape) < 0.05
        W_missing = W.copy()
        W_missing[mask] = np.nan
        groups = np.repeat(np.arange(500), 2)  # groups for dynamic dml

        # model that can handle missing values
        nuisance_model = make_pipeline(Imputer(strategy='mean'), LinearRegression())
        OrthoLearner(discrete_outcome=False, discrete_treatment=False, treatment_featurizer=None,
                     discrete_instrument=None, categories='auto', cv=3, random_state=1,
                     allow_missing=True).fit(y, T, W=W_missing)

        CausalForestDML(model_y=nuisance_model, model_t=nuisance_model,
                        allow_missing=True).fit(y, T, X=X, W=W_missing)

        DynamicDML(model_y=nuisance_model, model_t=nuisance_model,
                   allow_missing=True).fit(y, T, W=W_missing, groups=groups)

        LinearDML(model_y=nuisance_model, model_t=nuisance_model,
                  allow_missing=True).dowhy.fit(y, T, X=X, W=W_missing)
//  [python]
def test_missing(self):
        # create data with missing values
        np.random.seed(123)
        X = np.random.normal(size=(1000, 1))
        W = np.random.normal(size=(1000, 5))
        T = X[:, 0] + np.random.normal(size=(1000,))
        y = (1 + 0.5 * X[:, 0]) * T + X[:, 0] + np.random.normal(size=(1000,))
        mask = np.random.rand(*W.shape) < 0.05
        W_missing = W.copy()
        W_missing[mask] = np.nan
        groups = np.repeat(np.arange(500), 2)  # groups for dynamic dml

        # model that can handle missing values
        nuisance_model = make_pipeline(SimpleImputer(strategy='mean'), LinearRegression())
        OrthoLearner(discrete_outcome=False, discrete_treatment=False, treatment_featurizer=None,
                     discrete_instrument=None, categories='auto', cv=3, random_state=1,
                     allow_missing=True).fit(y, T, W=W_missing)

        CausalForestDML(model_y=nuisance_model, model_t=nuisance_model,
                        allow_missing=True).fit(y, T, X=X, W=W_missing)

        DynamicDML(model_y=nuisance_model, model_t=nuisance_model,
                   allow_missing=True).fit(y, T, W=W_missing, groups=groups)

        LinearDML(model_y=nuisance_model, model_t=nuisance_model,
                  allow_missing=True).dowhy.fit(y, T, X=X, W=W_missing)
//  [python]
def _corr_preprocess(da, sample_mch, sample_mcg, cpu=1):
    imputer = Imputer(copy=False)
    df = da.to_pandas()
    imputer.fit_transform(df)
    assert df.isna().values.sum() == 0

    adata = anndata.AnnData(
        X=df.values,
        obs=pd.DataFrame([], index=df.index),
        var=pd.DataFrame([], index=df.columns),
    )

    adata.var["pos"] = da["_pos"]

    # standardize global
    sample_mcg = (sample_mcg - sample_mcg.mean()) / sample_mcg.std()
    sample_mch = (sample_mch - sample_mch.mean()) / sample_mch.std()
    # add global
    adata.obs["sample_mch"] = sample_mch
    adata.obs["sample_mcg"] = sample_mcg

    # remove dmr that has 0 std
    # otherwise regress out will fail.
    adata = adata[:, adata.X.std(axis=0) > 0].copy()

    if adata.shape[1] > 0:
        # regress out global mCH and mCG
        # happens on each region
        sc.pp.regress_out(adata, keys=["sample_mch", "sample_mcg"], n_jobs=cpu)
        sc.pp.scale(adata)
    else:
        print("All features are removed due to 0 std")
    return adata
//  [python]
def _corr_preprocess(da, sample_mch, sample_mcg, cpu=1):
    imputer = SimpleImputer(copy=False)
    df = da.to_pandas()
    imputer.fit_transform(df)
    assert df.isna().values.sum() == 0

    adata = anndata.AnnData(
        X=df.values,
        obs=pd.DataFrame([], index=df.index),
        var=pd.DataFrame([], index=df.columns),
    )

    adata.var["pos"] = da["_pos"]

    # standardize global
    sample_mcg = (sample_mcg - sample_mcg.mean()) / sample_mcg.std()
    sample_mch = (sample_mch - sample_mch.mean()) / sample_mch.std()
    # add global
    adata.obs["sample_mch"] = sample_mch
    adata.obs["sample_mcg"] = sample_mcg

    # remove dmr that has 0 std
    # otherwise regress out will fail.
    adata = adata[:, adata.X.std(axis=0) > 0].copy()

    if adata.shape[1] > 0:
        # regress out global mCH and mCG
        # happens on each region
        sc.pp.regress_out(adata, keys=["sample_mch", "sample_mcg"], n_jobs=cpu)
        sc.pp.scale(adata)
    else:
        print("All features are removed due to 0 std")
    return adata
//  [python]
_logger = logging.getLogger(__name__)


price_pipe = Pipeline(
    [
        (
            "numerical_imputer",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.numerical_vars,
                transformer=Imputer(strategy="most_frequent"),
            ),
        ),
        (
            "categorical_imputer",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.categorical_vars,
                transformer=Imputer(strategy="constant", fill_value="missing"),
            ),
        ),
        (
            "temporal_variable",
            pp.TemporalVariableEstimator(
                variables=config.model_config.temporal_vars,
                reference_variable=config.model_config.drop_features,
            ),
        ),
        (
            "rare_label_encoder",
            RareLabelEncoder(
                tol=config.model_config.rare_label_tol,
                n_categories=config.model_config.rare_label_n_categories,
                variables=config.model_config.categorical_vars,
            ),
        ),
        (
            "categorical_encoder",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.categorical_vars,
                transformer=OrdinalEncoder(),
            ),
        ),
        (
            "drop_features",
            pp.DropUnecessaryFeatures(
                variables_to_drop=config.model_config.drop_features,
            ),
        ),
        (
            "gb_model",
            GradientBoostingRegressor(
                loss=config.model_config.loss,
                random_state=config.model_config.random_state,
                n_estimators=config.model_config.n_estimators,
            ),
        ),
    ]
)
//  [python]
_logger = logging.getLogger(__name__)


price_pipe = Pipeline(
    [
        (
            "numerical_imputer",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.numerical_vars,
                transformer=SimpleImputer(strategy="most_frequent"),
            ),
        ),
        (
            "categorical_imputer",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.categorical_vars,
                transformer=SimpleImputer(strategy="constant", fill_value="missing"),
            ),
        ),
        (
            "temporal_variable",
            pp.TemporalVariableEstimator(
                variables=config.model_config.temporal_vars,
                reference_variable=config.model_config.drop_features,
            ),
        ),
        (
            "rare_label_encoder",
            RareLabelEncoder(
                tol=config.model_config.rare_label_tol,
                n_categories=config.model_config.rare_label_n_categories,
                variables=config.model_config.categorical_vars,
            ),
        ),
        (
            "categorical_encoder",
            pp.SklearnTransformerWrapper(
                variables=config.model_config.categorical_vars,
                transformer=OrdinalEncoder(),
            ),
        ),
        (
            "drop_features",
            pp.DropUnecessaryFeatures(
                variables_to_drop=config.model_config.drop_features,
            ),
        ),
        (
            "gb_model",
            GradientBoostingRegressor(
                loss=config.model_config.loss,
                random_state=config.model_config.random_state,
                n_estimators=config.model_config.n_estimators,
            ),
        ),
    ]
)