jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived.values

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPClassifier(hidden_layer_sizes=(20, 20),
                                                           max_iter=400, random_state=0))])
        clf2 = Pipeline(steps=[('preprocessor', preprocessor),
                               ('classifier', MLPClassifier(hidden_layer_sizes=(50, 100, 50),
                                                            max_iter=400, random_state=0))])

        clf.fit(self.X, self.y)
        clf2.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, label="model1", verbose=False)
        self.exp2 = dx.Explainer(clf2, self.X, self.y, verbose=False)
        self.exp3 = dx.Explainer(clf, self.X, self.y, label="model3", verbose=False)
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived.values

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPClassifier(hidden_layer_sizes=(20, 20),
                                                           max_iter=400, random_state=0))])
        clf2 = Pipeline(steps=[('preprocessor', preprocessor),
                               ('classifier', MLPClassifier(hidden_layer_sizes=(50, 100, 50),
                                                            max_iter=400, random_state=0))])

        clf.fit(self.X, self.y)
        clf2.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, label="model1", verbose=False)
        self.exp2 = dx.Explainer(clf2, self.X, self.y, verbose=False)
        self.exp3 = dx.Explainer(clf, self.X, self.y, label="model3", verbose=False)
//  [python]
def prefit_processing(self, df):
        """
        Preprocessing dataframes before fitting to a model (e.g. sklearn.neighbors.NearestNeighbors) .  
        It consists of imputing missing values with most frequent strategy and converting categorical values to numerical.

        Parameters
        -----------
        df : pd.DataFrame

        Returns
        ----------
        df : pd.DataFrame
                Processed DataFrame
        """
        # Find columns with missing values and impute them
        for col in df.columns[df.isna().any()].tolist():
            imp = Imputer(missing_values=pd.NA, strategy='most_frequent')
            col_dtype = df[col].dtype
            if isinstance(col_dtype,pd.CategoricalDtype):
                col_dtype = "category"

            df[col] = imp.fit_transform(df[[col]])
            df[col] = df[col].astype(col_dtype)

        # Find columns containing categorical values and convert to numerical
        for col in df.select_dtypes(include=["object_","category"]).columns:
            ord_enc = OrdinalEncoder()
            df[col] = ord_enc.fit_transform(df[[col]])

        return df
//  [python]
def prefit_processing(self, df):
        """
        Preprocessing dataframes before fitting to a model (e.g. sklearn.neighbors.NearestNeighbors) .  
        It consists of imputing missing values with most frequent strategy and converting categorical values to numerical.

        Parameters
        -----------
        df : pd.DataFrame

        Returns
        ----------
        df : pd.DataFrame
                Processed DataFrame
        """
        # Find columns with missing values and impute them
        for col in df.columns[df.isna().any()].tolist():
            imp = SimpleImputer(missing_values=pd.NA, strategy='most_frequent')
            col_dtype = df[col].dtype
            if isinstance(col_dtype,pd.CategoricalDtype):
                col_dtype = "category"

            df[col] = imp.fit_transform(df[[col]])
            df[col] = df[col].astype(col_dtype)

        # Find columns containing categorical values and convert to numerical
        for col in df.select_dtypes(include=["object_","category"]).columns:
            ord_enc = OrdinalEncoder()
            df[col] = ord_enc.fit_transform(df[[col]])

        return df
//  [python]
def fit(self, x, y):
        """Fit model."""
        _, y = check_X_y(x, y, force_all_finite="allow-nan", y_numeric=True)
        if type(x) != pd.DataFrame:
            x = pd.DataFrame(np.asarray(x))
        self.feature_in_names_ = list(x.columns)
        self.n_features_in_ = x.shape[1]

        # Remove always null columns
        is_column_null = x.isnull().all(axis="index")
        self.non_null_columns_ = list(x.columns[~is_column_null])

        self.regressor_ = clone(self.base_estimator)

        # Build the proper imputation transformer
        # - Identity function if strategy is None
        # - SimpleImputer with the dedicated strategy
        if self.imputation_strategy is None:
            self.imputer_ = FunctionTransformer(func=self._identity)
        else:
            self.imputer_ = Imputer(
                missing_values=self.missing_values,
                strategy=self.imputation_strategy,
                fill_value=self.fill_value,
            )

        self.pipeline_ = Pipeline(
            [("imputer", self.imputer_), ("regressor", self.regressor_)]
        )

        # Fit only on non_null_columns
        self.pipeline_.fit(x[self.non_null_columns_], y)

        return self
//  [python]
def fit(self, x, y):
        """Fit model."""
        _, y = check_X_y(x, y, force_all_finite="allow-nan", y_numeric=True)
        if type(x) != pd.DataFrame:
            x = pd.DataFrame(np.asarray(x))
        self.feature_in_names_ = list(x.columns)
        self.n_features_in_ = x.shape[1]

        # Remove always null columns
        is_column_null = x.isnull().all(axis="index")
        self.non_null_columns_ = list(x.columns[~is_column_null])

        self.regressor_ = clone(self.base_estimator)

        # Build the proper imputation transformer
        # - Identity function if strategy is None
        # - SimpleImputer with the dedicated strategy
        if self.imputation_strategy is None:
            self.imputer_ = FunctionTransformer(func=self._identity)
        else:
            self.imputer_ = SimpleImputer(
                missing_values=self.missing_values,
                strategy=self.imputation_strategy,
                fill_value=self.fill_value,
            )

        self.pipeline_ = Pipeline(
            [("imputer", self.imputer_), ("regressor", self.regressor_)]
        )

        # Fit only on non_null_columns
        self.pipeline_.fit(x[self.non_null_columns_], y)

        return self
//  [python]
def create_preprocessor(impute_strategy, max_category_levels, unique_category_str, continuous_features, skewed_features, onehot_features, embed_features, bool_features):
    """ Creates sklearn ColumnTransformer that can be fit to training data to preprocess it for tabular neural network. """
    transformers = []  # order of various column transformers in this list is important!
    if continuous_features:
        continuous_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy=impute_strategy)),
            ('scaler', StandardScaler())])
        transformers.append( ('continuous', continuous_transformer, continuous_features) )
    if skewed_features:
        power_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy=impute_strategy)),
            ('quantile', QuantileTransformer(output_distribution='normal')) ])  # Or output_distribution = 'uniform'
        transformers.append( ('skewed', power_transformer, skewed_features) )
    if onehot_features:
        onehot_transformer = Pipeline(steps=[
            # TODO: Consider avoiding converting to string for improved memory efficiency
            ('to_str', FunctionTransformer(convert_df_dtype_to_str)),
            ('imputer', Imputer(strategy='constant', fill_value=unique_category_str)),
            ('onehot', OneHotMergeRaresHandleUnknownEncoder(max_levels=max_category_levels, sparse=False))])  # test-time unknown values will be encoded as all zeros vector
        transformers.append( ('onehot', onehot_transformer, onehot_features) )
    if embed_features:  # Ordinal transformer applied to convert to-be-embedded categorical features to integer levels
        ordinal_transformer = Pipeline(steps=[
            ('to_str', FunctionTransformer(convert_df_dtype_to_str)),
            ('imputer', Imputer(strategy='constant', fill_value=unique_category_str)),
            ('ordinal', OrdinalMergeRaresHandleUnknownEncoder(max_levels=max_category_levels))])  # returns 0-n when max_category_levels = n-1. category n is reserved for unknown test-time categories.
        transformers.append( ('ordinal', ordinal_transformer, embed_features) )
    return ColumnTransformer(transformers=transformers, remainder='passthrough')
//  [python]
def create_preprocessor(impute_strategy, max_category_levels, unique_category_str, continuous_features, skewed_features, onehot_features, embed_features, bool_features):
    """ Creates sklearn ColumnTransformer that can be fit to training data to preprocess it for tabular neural network. """
    transformers = []  # order of various column transformers in this list is important!
    if continuous_features:
        continuous_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy=impute_strategy)),
            ('scaler', StandardScaler())])
        transformers.append( ('continuous', continuous_transformer, continuous_features) )
    if skewed_features:
        power_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy=impute_strategy)),
            ('quantile', QuantileTransformer(output_distribution='normal')) ])  # Or output_distribution = 'uniform'
        transformers.append( ('skewed', power_transformer, skewed_features) )
    if onehot_features:
        onehot_transformer = Pipeline(steps=[
            # TODO: Consider avoiding converting to string for improved memory efficiency
            ('to_str', FunctionTransformer(convert_df_dtype_to_str)),
            ('imputer', SimpleImputer(strategy='constant', fill_value=unique_category_str)),
            ('onehot', OneHotMergeRaresHandleUnknownEncoder(max_levels=max_category_levels, sparse=False))])  # test-time unknown values will be encoded as all zeros vector
        transformers.append( ('onehot', onehot_transformer, onehot_features) )
    if embed_features:  # Ordinal transformer applied to convert to-be-embedded categorical features to integer levels
        ordinal_transformer = Pipeline(steps=[
            ('to_str', FunctionTransformer(convert_df_dtype_to_str)),
            ('imputer', SimpleImputer(strategy='constant', fill_value=unique_category_str)),
            ('ordinal', OrdinalMergeRaresHandleUnknownEncoder(max_levels=max_category_levels))])  # returns 0-n when max_category_levels = n-1. category n is reserved for unknown test-time categories.
        transformers.append( ('ordinal', ordinal_transformer, embed_features) )
    return ColumnTransformer(transformers=transformers, remainder='passthrough')
//  [python]
def _imputation(self, method: str):
        # remove ProteinGroups with only NA before
        protein_group_na = self.mat.columns[self.mat.isna().all()].tolist()

        if len(protein_group_na) > 0:
            self.mat = self.mat.drop(protein_group_na, axis=1)
            logging.info(
                f" {len(protein_group_na)} Protein Groups were removed due to missing values."
            )

        logging.info("Imputing data...")

        if method == "mean":
            imp = sklearn.preprocessing.Imputer(missing_values=np.nan, strategy="mean", axis=0)
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "median":
            imp = sklearn.preprocessing.Imputer(missing_values=np.nan, strategy="median", axis=0)
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "knn":
            # change for text
            method = "k-Nearest Neighbor"
            imp = sklearn.impute.KNNImputer(n_neighbors=2, weights="uniform")
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "randomforest":
            randomforest = sklearn.ensemble.RandomForestRegressor(
                max_depth=10,
                bootstrap=True,
                max_samples=0.5,
                n_jobs=2,
                random_state=0,
                verbose=0,  #  random forest takes a while print progress
            )
            imp = sklearn.impute.IterativeImputer(
                random_state=0, estimator=randomforest
            )

            # the random forest imputer doesnt work with float32/float16..
            #  so the values are multiplied and converted to integers
            array_multi_mio = self.mat.values * 1000000
            array_int = array_multi_mio.astype("int")

            imputation_array = imp.fit_transform(array_int)
            imputation_array = imputation_array / 1000000

        else:
            raise ValueError(
                "Imputation method: {method} is invalid."
                "Choose 'mean'. 'median', 'knn' (for k-nearest Neighbors) or "
                "'randomforest' for random forest imputation."
            )

        self.mat = pd.DataFrame(
            imputation_array, index=self.mat.index, columns=self.mat.columns
        )
        self.preprocessing_info.update({"Imputation": method})
//  [python]
def _imputation(self, method: str):
        # remove ProteinGroups with only NA before
        protein_group_na = self.mat.columns[self.mat.isna().all()].tolist()

        if len(protein_group_na) > 0:
            self.mat = self.mat.drop(protein_group_na, axis=1)
            logging.info(
                f" {len(protein_group_na)} Protein Groups were removed due to missing values."
            )

        logging.info("Imputing data...")

        if method == "mean":
            imp = sklearn.impute.SimpleImputer(missing_values=np.nan, strategy="mean", keep_empty_features=True)
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "median":
            imp = sklearn.impute.SimpleImputer(missing_values=np.nan, strategy="median", keep_empty_features=True)
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "knn":
            # change for text
            method = "k-Nearest Neighbor"
            imp = sklearn.impute.KNNImputer(n_neighbors=2, weights="uniform")
            imputation_array = imp.fit_transform(self.mat.values)

        elif method == "randomforest":
            randomforest = sklearn.ensemble.RandomForestRegressor(
                max_depth=10,
                bootstrap=True,
                max_samples=0.5,
                n_jobs=2,
                random_state=0,
                verbose=0,  #  random forest takes a while print progress
            )
            imp = sklearn.impute.IterativeImputer(
                random_state=0, estimator=randomforest
            )

            # the random forest imputer doesnt work with float32/float16..
            #  so the values are multiplied and converted to integers
            array_multi_mio = self.mat.values * 1000000
            array_int = array_multi_mio.astype("int")

            imputation_array = imp.fit_transform(array_int)
            imputation_array = imputation_array / 1000000

        else:
            raise ValueError(
                "Imputation method: {method} is invalid."
                "Choose 'mean'. 'median', 'knn' (for k-nearest Neighbors) or "
                "'randomforest' for random forest imputation."
            )

        self.mat = pd.DataFrame(
            imputation_array, index=self.mat.index, columns=self.mat.columns
        )
        self.preprocessing_info.update({"Imputation": method})
//  [python]
def benign_kmeans():
    print("Importing benign.csv data...")
    benign_h2o = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/benign.csv"))

    benign_sci = np.genfromtxt(pyunit_utils.locate("smalldata/logreg/benign.csv"), delimiter=",")
    # Impute missing values with column mean
    imp = Imputer(missing_values=np.nan, strategy="mean")
    benign_sci = imp.fit_transform(benign_sci)

    for i in range(1,7):
        print("H2O K-Means with " + str(i) + " clusters:")
        benign_h2o_km = H2OKMeansEstimator(k=i)
        benign_h2o_km.train(x=list(range(benign_h2o.ncol)), training_frame=benign_h2o)
        print("H2O centers")
        print(benign_h2o_km.centers())

        benign_sci_km = KMeans(n_clusters=i, init='k-means++', n_init=1)
        benign_sci_km.fit(benign_sci)
        print("sckit centers")
        print(benign_sci_km.cluster_centers_)
//  [python]
def benign_kmeans():
    print("Importing benign.csv data...")
    benign_h2o = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/benign.csv"))

    benign_sci = np.genfromtxt(pyunit_utils.locate("smalldata/logreg/benign.csv"), delimiter=",")
    # Impute missing values with column mean
    imp = SimpleImputer(missing_values=np.nan, strategy="mean")
    benign_sci = imp.fit_transform(benign_sci)

    for i in range(1,7):
        print("H2O K-Means with " + str(i) + " clusters:")
        benign_h2o_km = H2OKMeansEstimator(k=i)
        benign_h2o_km.train(x=list(range(benign_h2o.ncol)), training_frame=benign_h2o)
        print("H2O centers")
        print(benign_h2o_km.centers())

        benign_sci_km = KMeans(n_clusters=i, init='k-means++', n_init=1)
        benign_sci_km.fit(benign_sci)
        print("sckit centers")
        print(benign_sci_km.cluster_centers_)
//  [python]
def train(annotations_file,
          hyperparameters_json_file,
          output_prefix):
    logger.info('Reading annotations...')
    annotation_names_i, X_ni = read_annotations(annotations_file)
    logger.info(f'Annotations: {annotation_names_i}.')

    logger.info('Reading hyperparameters...')
    with open(hyperparameters_json_file) as json_file:
        hyperparameters_kwargs = json.load(json_file)
    logger.info('Hyperparameters:', hyperparameters_kwargs)

    logger.info('Imputing annotations...')
    imputer = sklearn.preprocessing.Imputer(strategy='median')
    imputed_X_ni = imputer.fit_transform(X_ni)

    # SimpleImputer will drop any features that are completely missing, resulting in different shapes for
    # imputed_X_ni and X_ni and misalignment of features when training and scoring downstream if not checked.
    # We externally check for and fail in the presence of any entirely missing features, but we do a redundant check here.
    assert imputed_X_ni.shape == X_ni.shape, \
        f'Shape of imputed annotations differs from shape of raw annotations; at least one feature is completely missing ' \
        f'and hence dropped during imputation.'

    logger.info(f'Training IsolationForest with {imputed_X_ni.shape[0]} training sites x {imputed_X_ni.shape[1]} annotations...')
    clf = sklearn.ensemble.IsolationForest(**hyperparameters_kwargs)
    clf.fit(imputed_X_ni)
    logger.info('Training complete.')

    def score_samples(test_annotation_names_i,
                      test_X_ni):
        assert np.array_equal(test_annotation_names_i, annotation_names_i), \
            f'Input annotation names ({test_annotation_names_i}) must be identical to those used to train the scorer ({annotation_names_i}).'
        return clf.score_samples(imputer.transform(test_X_ni)) # TODO sklearn's implementation is single-threaded, but this could perhaps be parallelized

    scorer_lambda = lambda test_annotation_names_i, test_X_ni: score_samples(test_annotation_names_i, test_X_ni)

    logger.info(f'Pickling scorer...')
    output_scorer_pkl_file = f'{output_prefix}.scorer.pkl'
    with open(output_scorer_pkl_file, 'wb') as f:
        dill.dump(scorer_lambda, f) # the dill package can be used to pickle lambda functions
    logger.info(f'Scorer pickled to {output_scorer_pkl_file}.')
//  [python]
def train(annotations_file,
          hyperparameters_json_file,
          output_prefix):
    logger.info('Reading annotations...')
    annotation_names_i, X_ni = read_annotations(annotations_file)
    logger.info(f'Annotations: {annotation_names_i}.')

    logger.info('Reading hyperparameters...')
    with open(hyperparameters_json_file) as json_file:
        hyperparameters_kwargs = json.load(json_file)
    logger.info('Hyperparameters:', hyperparameters_kwargs)

    logger.info('Imputing annotations...')
    imputer = sklearn.impute.SimpleImputer(strategy='median')
    imputed_X_ni = imputer.fit_transform(X_ni)

    # SimpleImputer will drop any features that are completely missing, resulting in different shapes for
    # imputed_X_ni and X_ni and misalignment of features when training and scoring downstream if not checked.
    # We externally check for and fail in the presence of any entirely missing features, but we do a redundant check here.
    assert imputed_X_ni.shape == X_ni.shape, \
        f'Shape of imputed annotations differs from shape of raw annotations; at least one feature is completely missing ' \
        f'and hence dropped during imputation.'

    logger.info(f'Training IsolationForest with {imputed_X_ni.shape[0]} training sites x {imputed_X_ni.shape[1]} annotations...')
    clf = sklearn.ensemble.IsolationForest(**hyperparameters_kwargs)
    clf.fit(imputed_X_ni)
    logger.info('Training complete.')

    def score_samples(test_annotation_names_i,
                      test_X_ni):
        assert np.array_equal(test_annotation_names_i, annotation_names_i), \
            f'Input annotation names ({test_annotation_names_i}) must be identical to those used to train the scorer ({annotation_names_i}).'
        return clf.score_samples(imputer.transform(test_X_ni)) # TODO sklearn's implementation is single-threaded, but this could perhaps be parallelized

    scorer_lambda = lambda test_annotation_names_i, test_X_ni: score_samples(test_annotation_names_i, test_X_ni)

    logger.info(f'Pickling scorer...')
    output_scorer_pkl_file = f'{output_prefix}.scorer.pkl'
    with open(output_scorer_pkl_file, 'wb') as f:
        dill.dump(scorer_lambda, f) # the dill package can be used to pickle lambda functions
    logger.info(f'Scorer pickled to {output_scorer_pkl_file}.')
//  [python]
def test_explain_model_string_classes(self, mimic_explainer):
        adult_census_income = retrieve_dataset('AdultCensusIncome.csv', skipinitialspace=True)
        X = adult_census_income.drop(['income'], axis=1)
        y = adult_census_income[['income']]
        features = X.columns.values.tolist()
        classes = y['income'].unique().tolist()
        pipe_cfg = {
            'num_cols': X.dtypes[X.dtypes == 'int64'].index.values.tolist(),
            'cat_cols': X.dtypes[X.dtypes == 'object'].index.values.tolist(),
        }
        num_pipe = Pipeline([
            ('num_imputer', Imputer(strategy='median')),
            ('num_scaler', StandardScaler())
        ])
        cat_pipe = Pipeline([
            ('cat_imputer', Imputer(strategy='constant', fill_value='?')),
            ('cat_encoder', OneHotEncoder(handle_unknown='ignore', sparse=False))
        ])
        feat_pipe = ColumnTransformer([
            ('num_pipe', num_pipe, pipe_cfg['num_cols']),
            ('cat_pipe', cat_pipe, pipe_cfg['cat_cols'])
        ])
        X_train = X.copy()
        y_train = y.copy()
        X_train.reset_index(drop=True, inplace=True)
        y_train.reset_index(drop=True, inplace=True)
        X_train = feat_pipe.fit_transform(X_train)
        model = SGDClassifier()
        model = model.fit(X_train, y_train['income'])
        model_task = ModelTask.Classification
        explainer = mimic_explainer(model, X.iloc[:1000], LinearExplainableModel,
                                    augment_data=True, max_num_of_augmentations=10,
                                    features=features, classes=classes, model_task=model_task,
                                    transformations=feat_pipe)
        global_explanation = explainer.explain_global(X.iloc[:1000])
        assert global_explanation.method == LINEAR_METHOD

        self._verify_predictions_and_replication_metric(explainer, X.iloc[:1000])
//  [python]
def test_explain_model_string_classes(self, mimic_explainer):
        adult_census_income = retrieve_dataset('AdultCensusIncome.csv', skipinitialspace=True)
        X = adult_census_income.drop(['income'], axis=1)
        y = adult_census_income[['income']]
        features = X.columns.values.tolist()
        classes = y['income'].unique().tolist()
        pipe_cfg = {
            'num_cols': X.dtypes[X.dtypes == 'int64'].index.values.tolist(),
            'cat_cols': X.dtypes[X.dtypes == 'object'].index.values.tolist(),
        }
        num_pipe = Pipeline([
            ('num_imputer', SimpleImputer(strategy='median')),
            ('num_scaler', StandardScaler())
        ])
        cat_pipe = Pipeline([
            ('cat_imputer', SimpleImputer(strategy='constant', fill_value='?')),
            ('cat_encoder', OneHotEncoder(handle_unknown='ignore', sparse=False))
        ])
        feat_pipe = ColumnTransformer([
            ('num_pipe', num_pipe, pipe_cfg['num_cols']),
            ('cat_pipe', cat_pipe, pipe_cfg['cat_cols'])
        ])
        X_train = X.copy()
        y_train = y.copy()
        X_train.reset_index(drop=True, inplace=True)
        y_train.reset_index(drop=True, inplace=True)
        X_train = feat_pipe.fit_transform(X_train)
        model = SGDClassifier()
        model = model.fit(X_train, y_train['income'])
        model_task = ModelTask.Classification
        explainer = mimic_explainer(model, X.iloc[:1000], LinearExplainableModel,
                                    augment_data=True, max_num_of_augmentations=10,
                                    features=features, classes=classes, model_task=model_task,
                                    transformations=feat_pipe)
        global_explanation = explainer.explain_global(X.iloc[:1000])
        assert global_explanation.method == LINEAR_METHOD

        self._verify_predictions_and_replication_metric(explainer, X.iloc[:1000])
//  [python]
def get_column_transformer(self):
        """
        Transforming categorical and numerical features.

        Returns
        -------
        Pipeline
            pipeline of column transformers.

        """
        if self.column_transformer is None:
            assert self.df_x is not None

            # select categorical and numerical features
            cat_ix = self.df_x.select_dtypes(include=["object", "bool"]).columns
            num_ix = self.df_x.select_dtypes(include=["int64", "float64"]).columns

            # get the column indices, since the drops mess up the column names
            cat_new_ix = [self.df_x.columns.get_loc(col) for col in cat_ix]
            num_new_ix = [self.df_x.columns.get_loc(col) for col in num_ix]

            # pipeline for categorical data
            cat_preprocessing = make_pipeline(
                Imputer(strategy="constant", fill_value="NA"),
                OneHotEncoder(handle_unknown="ignore"),
            )

            # pipeline for numerical data
            num_preprocessing = make_pipeline(
                Imputer(strategy="mean"), MinMaxScaler()
            )

            # combine both pipeline using a columnTransformer
            self.column_transformer = ColumnTransformer(
                [
                    ("num", num_preprocessing, num_new_ix),
                    ("cat", cat_preprocessing, cat_new_ix),
                ]
            )

        return self.column_transformer
//  [python]
def get_column_transformer(self):
        """
        Transforming categorical and numerical features.

        Returns
        -------
        Pipeline
            pipeline of column transformers.

        """
        if self.column_transformer is None:
            assert self.df_x is not None

            # select categorical and numerical features
            cat_ix = self.df_x.select_dtypes(include=["object", "bool"]).columns
            num_ix = self.df_x.select_dtypes(include=["int64", "float64"]).columns

            # get the column indices, since the drops mess up the column names
            cat_new_ix = [self.df_x.columns.get_loc(col) for col in cat_ix]
            num_new_ix = [self.df_x.columns.get_loc(col) for col in num_ix]

            # pipeline for categorical data
            cat_preprocessing = make_pipeline(
                SimpleImputer(strategy="constant", fill_value="NA"),
                OneHotEncoder(handle_unknown="ignore"),
            )

            # pipeline for numerical data
            num_preprocessing = make_pipeline(
                SimpleImputer(strategy="mean"), MinMaxScaler()
            )

            # combine both pipeline using a columnTransformer
            self.column_transformer = ColumnTransformer(
                [
                    ("num", num_preprocessing, num_new_ix),
                    ("cat", cat_preprocessing, cat_new_ix),
                ]
            )

        return self.column_transformer
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('pe', PredictionError(Lasso()))
        ])

        model.fit(X_train, y_train)
        model.score(X_test, y_test)
        model['pe'].finalize()
        self.assert_images_similar(model['pe'], tol=2.0)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('pe', PredictionError(Lasso()))
        ])

        model.fit(X_train, y_train)
        model.score(X_test, y_test)
        model['pe'].finalize()
        self.assert_images_similar(model['pe'], tol=2.0)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('pe', PredictionError(Lasso()))
        ])

        model.fit(X_train, y_train)
        model.score(X_test, y_test)
        model['pe'].finalize()
        self.assert_images_similar(model['pe'], tol=2.0)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('pe', PredictionError(Lasso()))
        ])

        model.fit(X_train, y_train)
        model.score(X_test, y_test)
        model['pe'].finalize()
        self.assert_images_similar(model['pe'], tol=2.0)
//  [python]
def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]):
    import warnings

    def warn(*args, **kwargs):
        pass

    warnings.warn = warn

    x, y, test_x, test_y = x.cpu().numpy(), y.cpu().long().numpy(), test_x.cpu().numpy(), test_y.cpu().long().numpy()

    if impute:
        imp_mean = Imputer(missing_values=np.nan, strategy='mean')
        imp_mean.fit(x)
        x, test_x = imp_mean.transform(x), imp_mean.transform(test_x)

    if one_hot:
        def make_pd_from_np(x):
            data = pd.DataFrame(x)
            for c in cat_features:
                data.iloc[:, c] = data.iloc[:, c].astype('int')
            return data
        x, test_x = make_pd_from_np(x),  make_pd_from_np(test_x)
        transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore',
                                        sparse_output=False), cat_features)], remainder="passthrough")
        transformer.fit(x)
        x, test_x = transformer.transform(x), transformer.transform(test_x)

    if standardize:
        scaler = MinMaxScaler()
        scaler.fit(x)
        x, test_x = scaler.transform(x), scaler.transform(test_x)

    return x, y, test_x, test_y
//  [python]
def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]):
    import warnings

    def warn(*args, **kwargs):
        pass

    warnings.warn = warn

    x, y, test_x, test_y = x.cpu().numpy(), y.cpu().long().numpy(), test_x.cpu().numpy(), test_y.cpu().long().numpy()

    if impute:
        imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
        imp_mean.fit(x)
        x, test_x = imp_mean.transform(x), imp_mean.transform(test_x)

    if one_hot:
        def make_pd_from_np(x):
            data = pd.DataFrame(x)
            for c in cat_features:
                data.iloc[:, c] = data.iloc[:, c].astype('int')
            return data
        x, test_x = make_pd_from_np(x),  make_pd_from_np(test_x)
        transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore',
                                        sparse_output=False), cat_features)], remainder="passthrough")
        transformer.fit(x)
        x, test_x = transformer.transform(x), transformer.transform(test_x)

    if standardize:
        scaler = MinMaxScaler()
        scaler.fit(x)
        x, test_x = scaler.transform(x), scaler.transform(test_x)

    return x, y, test_x, test_y
//  [python]
def transformer_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300, device='cpu', N_ensemble_configurations=3, classifier=None, onehot=False):
    from sklearn.feature_selection import SelectKBest
    from sklearn.preprocessing import Imputer

    from mothernet.prediction.tabpfn import TabPFNClassifier

    if onehot:
        ohe = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore', max_categories=10,
                                sparse_output=False), cat_features)], remainder=Imputer(strategy="constant", fill_value=0))
        ohe.fit(x)
        x, test_x = ohe.transform(x), ohe.transform(test_x)
        if x.shape[1] > 100:
            skb = SelectKBest(k=100).fit(x, y)
            x, test_x = skb.transform(x), skb.transform(test_x)

    if classifier is None:
        classifier = TabPFNClassifier(device=device, N_ensemble_configurations=N_ensemble_configurations)
    tick = time.time()
    classifier.fit(x, y)
    fit_time = time.time() - tick
    # print('Train data shape', x.shape, ' Test data shape', test_x.shape)
    tick = time.time()
    pred = classifier.predict_proba(test_x)
    inference_time = time.time() - tick
    times = {'fit_time': fit_time, 'inference_time': inference_time}
    metric = metric_used(test_y, pred)

    return metric, pred, times
//  [python]
def transformer_metric(x, y, test_x, test_y, cat_features, metric_used, max_time=300, device='cpu', N_ensemble_configurations=3, classifier=None, onehot=False):
    from sklearn.feature_selection import SelectKBest
    from sklearn.impute import SimpleImputer

    from mothernet.prediction.tabpfn import TabPFNClassifier

    if onehot:
        ohe = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore', max_categories=10,
                                sparse_output=False), cat_features)], remainder=SimpleImputer(strategy="constant", fill_value=0))
        ohe.fit(x)
        x, test_x = ohe.transform(x), ohe.transform(test_x)
        if x.shape[1] > 100:
            skb = SelectKBest(k=100).fit(x, y)
            x, test_x = skb.transform(x), skb.transform(test_x)

    if classifier is None:
        classifier = TabPFNClassifier(device=device, N_ensemble_configurations=N_ensemble_configurations)
    tick = time.time()
    classifier.fit(x, y)
    fit_time = time.time() - tick
    # print('Train data shape', x.shape, ' Test data shape', test_x.shape)
    tick = time.time()
    pred = classifier.predict_proba(test_x)
    inference_time = time.time() - tick
    times = {'fit_time': fit_time, 'inference_time': inference_time}
    metric = metric_used(test_y, pred)

    return metric, pred, times
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('lasso', Lasso())
        ])

        oz = PredictionError(model)
        oz.fit(X_train, y_train)
        oz.score(X_test, y_test)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lasso', Lasso())
        ])

        oz = PredictionError(model)
        oz.fit(X_train, y_train)
        oz.score(X_test, y_test)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def data_impute(form, strategy, fill=None):
    if strategy != 'constant':
        return Imputer(missing_values=form, strategy=strategy, copy=False)
    else:
        return Imputer(missing_values=form, strategy=strategy, fill_value=fill, copy=False)
//  [python]
def data_impute(form, strategy, fill=None):
    if strategy != 'constant':
        return SimpleImputer(missing_values=form, strategy=strategy, copy=False)
    else:
        return SimpleImputer(missing_values=form, strategy=strategy, fill_value=fill, copy=False)
//  [python]
def test_autolog_does_not_capture_runs_for_preprocessing_or_feature_manipulation_estimators():
    """
    Verifies that preprocessing and feature manipulation estimators, which represent data
    manipulation steps (e.g., normalization, label encoding) rather than ML models, do not
    produce runs when their fit_* operations are invoked independently of an ML pipeline
    """
    mlflow.sklearn.autolog()

    # Create a run using the MLflow client, which will be resumed via the fluent API,
    # in order to avoid setting fluent-level tags (e.g., source and user). Suppressing these
    # tags simplifies test validation logic
    client = MlflowClient()
    run_id = client.create_run(experiment_id=0).info.run_id

    from sklearn.compose import ColumnTransformer
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.feature_selection import VarianceThreshold
    from sklearn.preprocessing import Imputer, LabelEncoder, MinMaxScaler, Normalizer

    with mlflow.start_run(run_id=run_id):
        Normalizer().fit_transform(np.random.random((5, 5)))
        LabelEncoder().fit([1, 2, 2, 6])
        MinMaxScaler().fit_transform(50 * np.random.random((10, 10)))
        Imputer().fit_transform([[1, 2], [np.nan, 3], [7, 6]])
        TfidfVectorizer().fit_transform(
            [
                "MLflow is an end-to-end machine learning platform.",
                "MLflow enables me to systematize my ML experimentation",
            ]
        )
        VarianceThreshold().fit_transform([[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]])
        ColumnTransformer([("norm", Normalizer(), [0])]).fit_transform([[0]])

    params, metrics, tags, artifacts = get_run_data(run_id)
    assert len(params) == 0
    assert len(metrics) == 0
    assert len(tags) == 0
    assert len(artifacts) == 0
//  [python]
def test_autolog_does_not_capture_runs_for_preprocessing_or_feature_manipulation_estimators():
    """
    Verifies that preprocessing and feature manipulation estimators, which represent data
    manipulation steps (e.g., normalization, label encoding) rather than ML models, do not
    produce runs when their fit_* operations are invoked independently of an ML pipeline
    """
    mlflow.sklearn.autolog()

    # Create a run using the MLflow client, which will be resumed via the fluent API,
    # in order to avoid setting fluent-level tags (e.g., source and user). Suppressing these
    # tags simplifies test validation logic
    client = MlflowClient()
    run_id = client.create_run(experiment_id=0).info.run_id

    from sklearn.compose import ColumnTransformer
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.feature_selection import VarianceThreshold
    from sklearn.impute import SimpleImputer
    from sklearn.preprocessing import LabelEncoder, MinMaxScaler, Normalizer

    with mlflow.start_run(run_id=run_id):
        Normalizer().fit_transform(np.random.random((5, 5)))
        LabelEncoder().fit([1, 2, 2, 6])
        MinMaxScaler().fit_transform(50 * np.random.random((10, 10)))
        SimpleImputer().fit_transform([[1, 2], [np.nan, 3], [7, 6]])
        TfidfVectorizer().fit_transform(
            [
                "MLflow is an end-to-end machine learning platform.",
                "MLflow enables me to systematize my ML experimentation",
            ]
        )
        VarianceThreshold().fit_transform([[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]])
        ColumnTransformer([("norm", Normalizer(), [0])]).fit_transform([[0]])

    params, metrics, tags, artifacts = get_run_data(run_id)
    assert len(params) == 0
    assert len(metrics) == 0
    assert len(tags) == 0
    assert len(artifacts) == 0
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('lasso', Lasso())
        ])

        oz = prediction_error(model, X_train, y_train, X_test, y_test)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_concrete()
        X_train, X_test, y_train, y_test = tts(X, y, test_size=0.2, random_state=42)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lasso', Lasso())
        ])

        oz = prediction_error(model, X_train, y_train, X_test, y_test)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", Imputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)


# For categoricals, we:
# 1. Convert boolean values to strings
# 2. Impute missing values with the string "missing"
# 3. One hot encode the data (ignoring new categorical levels at prediction time)
# You can set `handle_unknown='error'` to make your model raise an error at prediction time if
# it encounters a new categorical level
categorical_pipeline = Pipeline(
    steps=[
        ("bool_to_string", FunctionTransformer(to_string)),
        ("imputer", Imputer(strategy="constant", fill_value="missing")),
        ("onehot", OneHotEncoder(handle_unknown="ignore")),
    ]
)
//  [python]
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)


# For categoricals, we:
# 1. Convert boolean values to strings
# 2. Impute missing values with the string "missing"
# 3. One hot encode the data (ignoring new categorical levels at prediction time)
# You can set `handle_unknown='error'` to make your model raise an error at prediction time if
# it encounters a new categorical level
categorical_pipeline = Pipeline(
    steps=[
        ("bool_to_string", FunctionTransformer(to_string)),
        ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
        ("onehot", OneHotEncoder(handle_unknown="ignore")),
    ]
)
//  [python]
def init_imputer(self, data):
        """
        since cases with ADD labels is only a subset of the cases with COG label
        in this function, we will initialize a single imputer
        and fit the imputer based on the COG cases from the training part
        """
        imputation_method = self.config['impute_method']
        if imputation_method == 'mean':
            imp = Imputer(missing_values='NaN', strategy='mean')
        elif imputation_method == 'median':
            imp = Imputer(missing_values='NaN', strategy='median')
        elif imputation_method == 'most_frequent':
            imp = Imputer(missing_values='NaN', strategy='most_frequent')
        elif imputation_method == 'constant':
            imp = Imputer(missing_values='NaN', strategy='constant')
        elif imputation_method == 'KNN':
            imp = KNNImputer(n_neighbors=20)
        elif imputation_method == 'Multivariate':
            imp = IterativeImputer(max_iter=1000)
        else:
            raise NameError('method for imputation not supported')
        imp.fit(data)
        return imp
//  [python]
def init_imputer(self, data):
        """
        since cases with ADD labels is only a subset of the cases with COG label
        in this function, we will initialize a single imputer
        and fit the imputer based on the COG cases from the training part
        """
        imputation_method = self.config['impute_method']
        if imputation_method == 'mean':
            imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        elif imputation_method == 'median':
            imp = SimpleImputer(missing_values=np.nan, strategy='median')
        elif imputation_method == 'most_frequent':
            imp = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
        elif imputation_method == 'constant':
            imp = SimpleImputer(missing_values=np.nan, strategy='constant')
        elif imputation_method == 'KNN':
            imp = KNNImputer(n_neighbors=20)
        elif imputation_method == 'Multivariate':
            imp = IterativeImputer(max_iter=1000)
        else:
            raise NameError('method for imputation not supported')
        imp.fit(data)
        return imp
//  [python]
num_pipeline = Pipeline(steps=[
    ('impute', Imputer(strategy='mean')),
    ('scale',MinMaxScaler())])

cat_pipeline = Pipeline(steps=[
        ('impute', Imputer(strategy='most_frequent')),
        ('one-hot',OneHotEncoder(handle_unknown='ignore', sparse=False))
    ])