jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def pd_ridgecv_feature_selector(
        df: pd.DataFrame,
        info_col_list: Optional[List[str]] = None,
        target: Union[str, List[str]] = 'target',
        treatment_col: Optional[Union[str, List[str]]] = None,
        weights_col_list: Optional[List[str]] = None,
        category_col_list: Optional[List[str]] = None,
        model: Optional = None) -> pd.DataFrame:
    """
    Processes input data and uses RidgeCV for feature selection.

    Args:
        df (pd.DataFrame): The input DataFrame.
        info_col_list (Optional[List[str]]): List of informational column names.
        target (Union[str, List[str]]): Target column name(s) for the model.
        treatment_col (Optional[Union[str, List[str]]): Treatment column name(s).
        weights_col_list (Optional[List[str]]): List of weights column names.
        category_col_list (Optional[List[str]]): List of categorical column names.
        model (Optional): The RidgeCV model to use. If None, a default model is used.

    Returns:
        pd.DataFrame: A DataFrame containing feature weights, their absolute values, and ranks.

    """
    feature_col_list, target_col_list, numeric_col_list, category_col_list, df = pd_input_preprocing(
        df,
        info_col_list=info_col_list,
        target=target,
        treatment=treatment_col,
        weights_col_list=weights_col_list,
        category_col_list=category_col_list
    )

    pipe = make_pipeline(
        Imputer(strategy='median'),
        StandardScaler(with_mean=False),
        RidgeCV(alphas=np.geomspace(1e5, 1e-5, num=100)) if model is None else model
    )

    target_data = df[target] if isinstance(target, str) else df[target_col_list]
    fitted_pipe = pipe.fit(df[feature_col_list].pipe(pd.get_dummies), target_data)

    result = pd.DataFrame(
        fitted_pipe[-1].coef_,
        index='"' + pd.Series(target) + '" weight',
        columns=fitted_pipe[:-1].get_feature_names_out(),
    ).T

    result.insert(0, 'rank', result.abs().max(axis=1).rank(ascending=False).astype('int'))
    result = result.sort_values('rank', ascending=True)

    return result
//  [python]
def pd_ridgecv_feature_selector(
        df: pd.DataFrame,
        info_col_list: Optional[List[str]] = None,
        target: Union[str, List[str]] = 'target',
        treatment_col: Optional[Union[str, List[str]]] = None,
        weights_col_list: Optional[List[str]] = None,
        category_col_list: Optional[List[str]] = None,
        model: Optional = None) -> pd.DataFrame:
    """
    Processes input data and uses RidgeCV for feature selection.

    Args:
        df (pd.DataFrame): The input DataFrame.
        info_col_list (Optional[List[str]]): List of informational column names.
        target (Union[str, List[str]]): Target column name(s) for the model.
        treatment_col (Optional[Union[str, List[str]]]): Treatment column name(s).
        weights_col_list (Optional[List[str]]): List of weights column names.
        category_col_list (Optional[List[str]]): List of categorical column names.
        model (Optional): The RidgeCV model to use. If None, a default model is used.

    Returns:
        pd.DataFrame: A DataFrame containing feature weights, their absolute values, and ranks.

    """
    feature_col_list, target_col_list, numeric_col_list, category_col_list, df = pd_input_preprocing(
        df,
        info_col_list=info_col_list,
        target=target,
        treatment=treatment_col,
        weights_col_list=weights_col_list,
        category_col_list=category_col_list
    )

    pipe = make_pipeline(
        SimpleImputer(strategy='median'),
        StandardScaler(with_mean=False),
        RidgeCV(alphas=np.geomspace(1e5, 1e-5, num=100)) if model is None else model
    )

    target_data = df[target] if isinstance(target, str) else df[target_col_list]
    fitted_pipe = pipe.fit(df[feature_col_list].pipe(pd.get_dummies), target_data)

    result = pd.DataFrame(
        fitted_pipe[-1].coef_,
        index='"' + pd.Series(target) + '" weight',
        columns=fitted_pipe[:-1].get_feature_names_out(),
    ).T

    result.insert(0, 'rank', result.abs().max(axis=1).rank(ascending=False).astype('int'))
    result = result.sort_values('rank', ascending=True)

    return result
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('lc',
             LearningCurve(MultinomialNB(), cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42))
        ])

        model.fit(X, y)
        model['lc'].finalize()
        self.assert_images_similar(model['lc'], tol=2.0)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lc',
             LearningCurve(MultinomialNB(), cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42))
        ])

        model.fit(X, y)
        model['lc'].finalize()
        self.assert_images_similar(model['lc'], tol=2.0)
//  [python]
def impute(df, config):
    if not (config.get("missing_data_label", False)):
        return df, None
    else:
        imp = Imputer(missing_values=config.missing_data_label, strategy="mean")
        mask = df.applymap(lambda x: True if x == config.missing_data_label else False)
        data = df.values
        col_mask = (data == config.missing_data_label).all(axis=0)
        data[:, ~col_mask] = imp.fit_transform(data)
        return data, mask
//  [python]
def impute(df, config):
    if not (config.get("missing_data_label", False)):
        return df, None
    else:
        imp = SimpleImputer(missing_values=config.missing_data_label, strategy="mean")
        mask = df.applymap(lambda x: True if x == config.missing_data_label else False)
        data = df.values
        col_mask = (data == config.missing_data_label).all(axis=0)
        data[:, ~col_mask] = imp.fit_transform(data)
        return data, mask
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('lc', learning_curve(MultinomialNB(), X, y, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4,
                                  random_state=42))
        ])
        model['lc'].finalize()
        self.assert_images_similar(model['lc'], tol=2.0)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lc', learning_curve(MultinomialNB(), X, y, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4,
                                  random_state=42))
        ])
        model['lc'].finalize()
        self.assert_images_similar(model['lc'], tol=2.0)
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('nb', MultinomialNB())
        ])

        oz = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42)
        oz.fit(X, y)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('nb', MultinomialNB())
        ])

        oz = LearningCurve(model, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42)
        oz.fit(X, y)
        oz.finalize()
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        within a quickmethod
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('nb', MultinomialNB())
        ])

        oz = learning_curve(model, X, y, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42)
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        within a quickmethod
        """
        X, y = load_game()

        # Encode the categorical data
        X = OneHotEncoder().fit_transform(X)
        y = LabelEncoder().fit_transform(y)

        # Create the learning curve visualizer
        cv = StratifiedKFold(n_splits=12)
        sizes = np.linspace(0.3, 1.0, 10)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('nb', MultinomialNB())
        ])

        oz = learning_curve(model, X, y, cv=cv, scoring='f1_weighted', train_sizes=sizes, n_jobs=4, random_state=42)
        self.assert_images_similar(oz, tol=2.0)
//  [python]
def encode(data, dtype=numpy.int32, missing_values='', unknown_value=-1):
    """
    把方言读音编码为整数.

    Parameters:
        data (`pandas.DataFrame`): M x N 字符串矩阵，每行为一个字，
            每列为一个方言点的声母/韵母/声调，空串代表空值

    Returns:
        codes (`numpy.ndarray`): M x N 整数矩阵，空值为 -1
        categories (list of `numpy.ndarray`): 长度为 N 的列表，每个元素是每一列的类别
    """

    encoder = sklearn.preprocessing.OrdinalEncoder(
        dtype=dtype,
        handle_unknown='use_encoded_value',
        unknown_value=unknown_value
    )
    encoder.fit(
        # 为了让编码器正常工作，先补全缺失特征
        sklearn.preprocessing.Imputer(
            missing_values=missing_values,
            strategy='most_frequent'
        ).fit_transform(data)
    )

    return encoder.transform(data), encoder.categories_
//  [python]
def encode(data, dtype=numpy.int32, missing_values='', unknown_value=-1):
    """
    把方言读音编码为整数.

    Parameters:
        data (`pandas.DataFrame`): M x N 字符串矩阵，每行为一个字，
            每列为一个方言点的声母/韵母/声调，空串代表空值

    Returns:
        codes (`numpy.ndarray`): M x N 整数矩阵，空值为 -1
        categories (list of `numpy.ndarray`): 长度为 N 的列表，每个元素是每一列的类别
    """

    encoder = sklearn.preprocessing.OrdinalEncoder(
        dtype=dtype,
        handle_unknown='use_encoded_value',
        unknown_value=unknown_value
    )
    encoder.fit(
        # 为了让编码器正常工作，先补全缺失特征
        sklearn.impute.SimpleImputer(
            missing_values=missing_values,
            strategy='most_frequent'
        ).fit_transform(data)
    )

    return encoder.transform(data), encoder.categories_
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()

        # Create a list of alphas to cross-validate against
        alphas = np.logspace(-10, 1, 400)

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('alpha', AlphaSelection(LassoCV(random_state=42, alphas=alphas)))
        ])

        model.fit(X, y)
        model['alpha'].finalize()
        self.assert_images_similar(model['alpha'], tol=2.0)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()

        # Create a list of alphas to cross-validate against
        alphas = np.logspace(-10, 1, 400)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('alpha', AlphaSelection(LassoCV(random_state=42, alphas=alphas)))
        ])

        model.fit(X, y)
        model['alpha'].finalize()
        self.assert_images_similar(model['alpha'], tol=2.0)
//  [python]
def test_manual_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()

        # Create a list of alphas to cross-validate against
        alpha_values = np.logspace(1, 4, 50)

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('alpha', ManualAlphaSelection(Ridge(random_state=42), alphas=alpha_values, cv=12,
                                           scoring="neg_mean_squared_error"))
        ])

        model.fit(X, y)
        model['alpha'].finalize()
        self.assert_images_similar(model['alpha'], tol=2.0)
//  [python]
def test_manual_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_concrete()

        # Create a list of alphas to cross-validate against
        alpha_values = np.logspace(1, 4, 50)

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('alpha', ManualAlphaSelection(Ridge(random_state=42), alphas=alpha_values, cv=12,
                                           scoring="neg_mean_squared_error"))
        ])

        model.fit(X, y)
        model['alpha'].finalize()
        self.assert_images_similar(model['alpha'], tol=2.0)
//  [python]
def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]):
    import warnings
    def warn(*args, **kwargs):
        pass

    warnings.warn = warn

    x, y, test_x, test_y = x.cpu().numpy(), y.cpu().long().numpy(), test_x.cpu().numpy(), test_y.cpu().long().numpy()

    if impute:
        imp_mean = Imputer(missing_values=np.nan, strategy='mean')
        imp_mean.fit(x)
        x, test_x = imp_mean.transform(x), imp_mean.transform(test_x)

    if one_hot:
        def make_pd_from_np(x):
            data = pd.DataFrame(x)
            for c in cat_features:
                data.iloc[:, c] = data.iloc[:, c].astype('int')
            return data
        x, test_x = make_pd_from_np(x),  make_pd_from_np(test_x)
        transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore', sparse=False), cat_features)], remainder="passthrough")
        transformer.fit(x)
        x, test_x = transformer.transform(x), transformer.transform(test_x)

    if standardize:
        scaler = MinMaxScaler()
        scaler.fit(x)
        x, test_x = scaler.transform(x), scaler.transform(test_x)

    return x, y, test_x, test_y
//  [python]
def preprocess_impute(x, y, test_x, test_y, impute, one_hot, standardize, cat_features=[]):
    import warnings
    def warn(*args, **kwargs):
        pass

    warnings.warn = warn

    x, y, test_x, test_y = x.cpu().numpy(), y.cpu().long().numpy(), test_x.cpu().numpy(), test_y.cpu().long().numpy()

    if impute:
        imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
        imp_mean.fit(x)
        x, test_x = imp_mean.transform(x), imp_mean.transform(test_x)

    if one_hot:
        def make_pd_from_np(x):
            data = pd.DataFrame(x)
            for c in cat_features:
                data.iloc[:, c] = data.iloc[:, c].astype('int')
            return data
        x, test_x = make_pd_from_np(x),  make_pd_from_np(test_x)
        transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(handle_unknown='ignore', sparse=False), cat_features)], remainder="passthrough")
        transformer.fit(x)
        x, test_x = transformer.transform(x), transformer.transform(test_x)

    if standardize:
        scaler = MinMaxScaler()
        scaler.fit(x)
        x, test_x = scaler.transform(x), scaler.transform(test_x)

    return x, y, test_x, test_y
//  [python]
def __init__(self, f_df, discrete=False):
        self.f_df = f_df
        self.discrete = discrete
        self.label_enc = preprocessing.OneHotEncoder(categories='auto')
        self.me_discretizer = MinimalEntropyDiscretizer()
        self.feature_enc = preprocessing.OneHotEncoder(categories='auto')
        self.imp = Imputer(missing_values=np.nan, strategy='mean')
        self.X_fname = None
        self.y_fname = None
//  [python]
def __init__(self, f_df, discrete=False):
        self.f_df = f_df
        self.discrete = discrete
        self.label_enc = preprocessing.OneHotEncoder(categories='auto')
        self.me_discretizer = MinimalEntropyDiscretizer()
        self.feature_enc = preprocessing.OneHotEncoder(categories='auto')
        self.imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        self.X_fname = None
        self.y_fname = None
//  [python]
def handle_filling(preprocess, df, target):
    result = []
    null_indicies = None 
    if preprocess['fill'] == 'ALL':
        null_indicies = df.index[df.isnull().any(axis=1)].tolist()
        df = df.dropna()
    else:
        if 'column' not in preprocess['fill']:
            raise Exception("A target column should be specified with the -- column -- command.")
        if 'target' not in preprocess['fill']:
            target = 'nan'
        else:
            target = preprocess['fill']['target']

        if 'tactic' not in preprocess['fill']:
            tactic = 'mean'
        else:
            tactic = preprocess['fill']['tactic']

        columns = preprocess['fill']['column']
        if isinstance(columns, str):
            columns = [columns]

        for column in columns:
            column_transformed = np.array(df[column]).reshape(-1, 1)
            imputer = Imputer(missing_values=(np.nan if target == 'nan' else target),
                                    strategy=tactic)
            data = np.array(df[column]).reshape(-1, 1)
            fitted = imputer.fit(data)
            df[column] = fitted.transform(data)
            if(column == target):
                result.append(fitted)

    result.append(df)
    result.append(null_indicies)
    return result
//  [python]
def handle_filling(preprocess, df, target):
    result = []
    null_indicies = None 
    if preprocess['fill'] == 'ALL':
        null_indicies = df.index[df.isnull().any(axis=1)].tolist()
        df = df.dropna()
    else:
        if 'column' not in preprocess['fill']:
            raise Exception("A target column should be specified with the -- column -- command.")
        if 'target' not in preprocess['fill']:
            target = 'nan'
        else:
            target = preprocess['fill']['target']

        if 'tactic' not in preprocess['fill']:
            tactic = 'mean'
        else:
            tactic = preprocess['fill']['tactic']

        columns = preprocess['fill']['column']
        if isinstance(columns, str):
            columns = [columns]

        for column in columns:
            column_transformed = np.array(df[column]).reshape(-1, 1)
            imputer = SimpleImputer(missing_values=(np.nan if target == 'nan' else target),
                                    strategy=tactic)
            data = np.array(df[column]).reshape(-1, 1)
            fitted = imputer.fit(data)
            df[column] = fitted.transform(data)
            if(column == target):
                result.append(fitted)

    result.append(df)
    result.append(null_indicies)
    return result
//  [python]
def test_sequence_transform(self):
		X = DataFrame([[None], [1], [None]], columns = ["a"])
		mapper = DataFrameMapper([
			(["a"], [ExpressionTransformer("0 if pandas.isnull(X[0]) else X[0]"), Imputer(missing_values = 0)])
		])
		Xt = mapper.fit_transform(X)
		self.assertEqual([[1], [1], [1]], Xt.tolist())
//  [python]
def test_sequence_transform(self):
		X = DataFrame([[None], [1], [None]], columns = ["a"])
		mapper = DataFrameMapper([
			(["a"], [ExpressionTransformer("0 if pandas.isnull(X[0]) else X[0]"), SimpleImputer(missing_values = 0)])
		])
		Xt = mapper.fit_transform(X)
		self.assertEqual([[1], [1], [1]], Xt.tolist())
//  [python]
train_data = pd.read_csv("./input/train.csv")
test_data = pd.read_csv("./input/test.csv")

# Separate target from predictors
y = train_data.SalePrice
X = train_data.drop(["SalePrice"], axis=1)

# Preprocessing for numerical data
numerical_transformer = Imputer(strategy="median")
//  [python]
train_data = pd.read_csv("./input/train.csv")
test_data = pd.read_csv("./input/test.csv")

# Separate target from predictors
y = train_data.SalePrice
X = train_data.drop(["SalePrice"], axis=1)

# Preprocessing for numerical data
numerical_transformer = SimpleImputer(strategy="median")
//  [python]
def embedding_preprocessor(data, text):
    full_pipeline = ColumnTransformer([], remainder="passthrough")
    if len(text) != 0:
        # Each text col needs a separate pipeline
        for x in range(len(text)):
            full_pipeline.transformers.append(
                (f"text_{x}",
                 Pipeline(
                     [
                         ('test',
                          FunctionTransformer(
                              lambda x: np.reshape(
                                  x.to_numpy(),
                                  (-1,
                                   1)))),
                         ('imputer',
                          Imputer(
                              strategy="constant",
                              fill_value="")),
                         ('raveler',
                          FunctionTransformer(
                              lambda x: x.ravel(),
                              accept_sparse=True)),
                         ('vect',
                          TfidfVectorizer()),
                         ('densifier',
                          FunctionTransformer(
                              lambda x: x.todense(),
                              accept_sparse=True)),
                         ('embedder',
                          FunctionTransformer(
                              textembedder,
                              accept_sparse=True))]),
                    text[x]))


    data_dict = {}
    data_dict['train'] = data
    train = full_pipeline.fit_transform(data_dict['train'])
    # test = full_pipeline.transform(data['test'])


    # Ternary clause because when running housing.csv,
    # the product of preprocessing is np array, but not when using landslide
    # data... not sure why
    final_data = pd.DataFrame(
        (train.toarray() if not isinstance(
            train,
            np.ndarray) else train),
        columns=data.columns)

    return final_data
//  [python]
def embedding_preprocessor(data, text):
    full_pipeline = ColumnTransformer([], remainder="passthrough")
    if len(text) != 0:
        # Each text col needs a separate pipeline
        for x in range(len(text)):
            full_pipeline.transformers.append(
                (f"text_{x}",
                 Pipeline(
                     [
                         ('test',
                          FunctionTransformer(
                              lambda x: np.reshape(
                                  x.to_numpy(),
                                  (-1,
                                   1)))),
                         ('imputer',
                          SimpleImputer(
                              strategy="constant",
                              fill_value="")),
                         ('raveler',
                          FunctionTransformer(
                              lambda x: x.ravel(),
                              accept_sparse=True)),
                         ('vect',
                          TfidfVectorizer()),
                         ('densifier',
                          FunctionTransformer(
                              lambda x: x.todense(),
                              accept_sparse=True)),
                         ('embedder',
                          FunctionTransformer(
                              textembedder,
                              accept_sparse=True))]),
                    text[x]))


    data_dict = {}
    data_dict['train'] = data
    train = full_pipeline.fit_transform(data_dict['train'])
    # test = full_pipeline.transform(data['test'])


    # Ternary clause because when running housing.csv,
    # the product of preprocessing is np array, but not when using landslide
    # data... not sure why
    final_data = pd.DataFrame(
        (train.toarray() if not isinstance(
            train,
            np.ndarray) else train),
        columns=data.columns)

    return final_data
//  [python]
def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.imputer = Imputer(strategy="mean")
//  [python]
def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.imputer = SimpleImputer(strategy="mean")
//  [python]
def impute(self):

        def save(imputer):
            file_dir = os.path.join(
                self.base_dir, 'Files', 'Transformers', 'Imputers')
            imputer_name = self.imputer_name + '.pkl'
            joblib.dump(imputer, os.path.join(file_dir, imputer_name))

        columns = self.fight_bouts.columns
        imputer = Imputer(strategy='most_frequent')
        imputer.fit(self.fight_bouts)
        imputed_data = imputer.transform(self.fight_bouts)
        save(imputer)

        self.fight_bouts = pd.DataFrame(imputed_data, columns=columns)
//  [python]
def impute(self):

        def save(imputer):
            file_dir = os.path.join(
                self.base_dir, 'Files', 'Transformers', 'Imputers')
            imputer_name = self.imputer_name + '.pkl'
            joblib.dump(imputer, os.path.join(file_dir, imputer_name))

        columns = self.fight_bouts.columns
        imputer = SimpleImputer(strategy='most_frequent', copy=False)
        imputer.fit(self.fight_bouts)
        imputed_data = imputer.transform(self.fight_bouts)
        save(imputer)

        self.fight_bouts = pd.DataFrame(imputed_data, columns=columns)
//  [python]
numerical_columns = [
    name for name in X_train.columns if "_calc_" in name and "_bin" not in name
]
numerical_pipeline = make_pipeline(
    FunctionTransformer(func=convert_float64, validate=False), StandardScaler()
)

categorical_columns = [name for name in X_train.columns if "_cat" in name]
categorical_pipeline = make_pipeline(
    Imputer(missing_values=-1, strategy="most_frequent"),
    OneHotEncoder(categories="auto"),
)
//  [python]
numerical_columns = [
    name for name in X_train.columns if "_calc_" in name and "_bin" not in name
]
numerical_pipeline = make_pipeline(
    FunctionTransformer(func=convert_float64, validate=False), StandardScaler()
)

categorical_columns = [name for name in X_train.columns if "_cat" in name]
categorical_pipeline = make_pipeline(
    SimpleImputer(missing_values=-1, strategy="most_frequent"),
    OneHotEncoder(categories="auto"),
)
//  [python]
def table_vectorizer() -> TableVectorizer:
    """Robust TableVectorizer."""
    # Fix missing values passing through.
    numerical_transformer = make_union(
        Imputer(missing_values=np.nan, strategy="median"),
        MissingIndicator(
            missing_values=np.nan,
            features="missing-only",
            sparse=False,
            error_on_new=False,
        ),
    )
    # Fix OHE generating warnings for unknown categories.
    ohe = OneHotEncoder(drop=None, handle_unknown="infrequent_if_exist")
    # Create the TableVectorizer.
    return TableVectorizer(
        numerical_transformer=numerical_transformer, low_cardinality_transformer=ohe
    )
//  [python]
def table_vectorizer() -> TableVectorizer:
    """Robust TableVectorizer."""
    # Fix missing values passing through.
    numerical_transformer = make_union(
        SimpleImputer(missing_values=np.nan, strategy="median"),
        MissingIndicator(
            missing_values=np.nan,
            features="missing-only",
            sparse=False,
            error_on_new=False,
        ),
    )
    # Fix OHE generating warnings for unknown categories.
    ohe = OneHotEncoder(drop=None, handle_unknown="infrequent_if_exist")
    # Create the TableVectorizer.
    return TableVectorizer(
        numerical_transformer=numerical_transformer, low_cardinality_transformer=ohe
    )
//  [python]
X, y = fetch_openml(
    "titanic", version=1, as_frame=True, return_X_y=True, parser="pandas"
)
numeric_features = ["age", "fare"]
numeric_transformer = make_pipeline(Imputer(strategy="median"), StandardScaler())
//  [python]
X, y = fetch_openml(
    "titanic", version=1, as_frame=True, return_X_y=True, parser="pandas"
)
numeric_features = ["age", "fare"]
numeric_transformer = make_pipeline(SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def train_model(artifacts_folder: str, data: AdultData):
    ordinal_features = [x for x in range(len(data.feature_names)) if x not in list(data.category_map.keys())]
    ordinal_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )
    categorical_features = list(data.category_map.keys())
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", ordinal_transformer, ordinal_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    clf = RandomForestClassifier(n_estimators=50)
    model = Pipeline(steps=[("preprocess", preprocessor), ("model", clf)])
    model.fit(data.X_train, data.Y_train)

    print("Train accuracy: ", accuracy_score(data.Y_train, model.predict(data.X_train)))
    print("Test accuracy: ", accuracy_score(data.Y_test, model.predict(data.X_test)))

    dump(model, f"{artifacts_folder}/{MODEL_FOLDER}" + "/model.joblib")
    return model
//  [python]
def train_model(artifacts_folder: str, data: AdultData):
    ordinal_features = [x for x in range(len(data.feature_names)) if x not in list(data.category_map.keys())]
    ordinal_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )
    categorical_features = list(data.category_map.keys())
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", ordinal_transformer, ordinal_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    clf = RandomForestClassifier(n_estimators=50)
    model = Pipeline(steps=[("preprocess", preprocessor), ("model", clf)])
    model.fit(data.X_train, data.Y_train)

    print("Train accuracy: ", accuracy_score(data.Y_train, model.predict(data.X_train)))
    print("Test accuracy: ", accuracy_score(data.Y_test, model.predict(data.X_test)))

    dump(model, f"{artifacts_folder}/{MODEL_FOLDER}" + "/model.joblib")
    return model
//  [python]
def fit_transform(self, X, y=None, **fit_params):
        from sklearn.decomposition import PCA
        from sklearn.preprocessing import Imputer

        X = X.to_numpy()
        imp = Imputer()
        X = imp.fit_transform(X)
        n_components = self._n_components
        if min(X.shape) <= n_components:
            n_components = min(X.shape) - 1

        self.pca = PCA(n_components=n_components)
        self.pca.fit(X)

        return self.pca.transform(X)
//  [python]
def fit_transform(self, X, y=None, **fit_params):
        from sklearn.decomposition import PCA
        from sklearn.impute import SimpleImputer

        X = X.to_numpy()
        imp = SimpleImputer()
        X = imp.fit_transform(X)
        n_components = self._n_components
        if min(X.shape) <= n_components:
            n_components = min(X.shape) - 1

        self.pca = PCA(n_components=n_components)
        self.pca.fit(X)

        return self.pca.transform(X)
//  [python]
def cluster_sample (cls, k, sample):
        df = pd.DataFrame(sample)

        # impute missing values using the column means (i.e., the avg
        # rating per item)
        # https://scikit-learn.org/stable/modules/impute.html
        imp = Imputer(missing_values=np.nan, strategy="mean")
        imp.fit(df.values)
        X = imp.transform(df.values).T

        # perform K-means clustering
        # https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
        km = KMeans(n_clusters=k)
        km.fit(X)
        centers = km.cluster_centers_

        # segment the items by their respective cluster labels
        clusters = defaultdict(set)
        labels = km.labels_

        for i in range(len(labels)):
            clusters[labels[i]].add(i)

        return centers.tolist(), dict(clusters)
//  [python]
def cluster_sample (cls, k, sample):
        df = pd.DataFrame(sample)

        # impute missing values using the column means (i.e., the avg
        # rating per item)
        # https://scikit-learn.org/stable/modules/impute.html
        imp = SimpleImputer(missing_values=np.nan, strategy="mean")
        imp.fit(df.values)
        X = imp.transform(df.values).T

        # perform K-means clustering
        # https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
        km = KMeans(n_clusters=k)
        km.fit(X)
        centers = km.cluster_centers_

        # segment the items by their respective cluster labels
        clusters = defaultdict(set)
        labels = km.labels_

        for i in range(len(labels)):
            clusters[labels[i]].add(i)

        return centers.tolist(), dict(clusters)
//  [python]
with wrap.importer():
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.preprocessing import Imputer as s
    from sklearn.linear_model import LogisticRegression
    from sklearn.naive_bayes import BernoulliNB
    from sklearn.preprocessing import Binarizer, OneHotEncoder


FH_RFC = OneHotEncoder(handle_unknown='ignore') >> RandomForestClassifier(n_estimators=20, n_jobs=4, max_depth=3)
BIN_BAYES = Binarizer(threshold=0.63) >> BernoulliNB(alpha=1.1)

STACK = ensemble.FullStack(FH_RFC, BIN_BAYES, crossvalidator=model_selection.StratifiedKFold(n_splits=2))

PIPELINE = s(strategy='mean') >> STACK >> LogisticRegression(max_iter=50, solver='lbfgs')
//  [python]
with wrap.importer():
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.impute import SimpleImputer
    from sklearn.linear_model import LogisticRegression
    from sklearn.naive_bayes import BernoulliNB
    from sklearn.preprocessing import Binarizer, OneHotEncoder


FH_RFC = OneHotEncoder(handle_unknown='ignore') >> RandomForestClassifier(n_estimators=20, n_jobs=4, max_depth=3)
BIN_BAYES = Binarizer(threshold=0.63) >> BernoulliNB(alpha=1.1)

STACK = ensemble.FullStack(FH_RFC, BIN_BAYES, crossvalidator=model_selection.StratifiedKFold(n_splits=2))

PIPELINE = SimpleImputer(strategy='mean') >> STACK >> LogisticRegression(max_iter=50, solver='lbfgs')
//  [python]
def preprocess(train_df, test_df, features, val_df=None, logger=None, objective='classification'):
    """
    The bulk of the preprocessing for a dataset goes here.
    """

    # display datasets
    if logger:
        logger.info('\ntrain_df:\n{}\n{}'.format(train_df.head(5), train_df.shape))
        logger.info('test_df:\n{}\n{}'.format(test_df.head(5), test_df.shape))

    # count number of NAN values per column
    if logger:
        logger.info('')
        for c in train_df.columns:
            logger.info('[TRAIN] {}, no. missing: {:,}'.format(c, train_df[c].isna().sum()))
            logger.info('[TEST] {}, no. missing: {:,}'.format(c, test_df[c].isna().sum()))

    # display column info
    if logger:
        logger.info('\ntrain_df column info:')
        for c in train_df.columns:
            logger.info(f'{c}: {train_df[c].dtype}, {len(train_df[c].unique())}')

    # categorize attributes
    label = features['label']
    numeric = features['numeric']
    categorical = features['categorical']

    numeric_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='median'))]
    )

    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))]
    )

    # time transforms
    start = time.time()

    # perform one-hot encoding for all cat. attributes
    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric),
            ('cat', categorical_transformer, categorical),
        ],
        sparse_threshold=0
    )

    # encode features
    train = preprocessor.fit_transform(train_df)
    test = preprocessor.transform(test_df)
    if val_df is not None:
        val = preprocessor.transform(val_df)

    # encode labels
    if objective == 'classification':
        le = LabelEncoder()
        train_label = le.fit_transform(train_df[label].values.ravel())
        test_label = le.transform(test_df[label].values.ravel())
        if val_df is not None:
            val_label = le.transform(val_df[label].values.ravel())

    # regression
    else:
        assert objective == 'regression'
        train_label = train_df[label].values.ravel()
        test_label = test_df[label].values.ravel()
        if val_df is not None:
            val_label = val_df[label].values.ravel()

    if logger:
        logger.info('transforming features...{:.3f}s'.format(time.time() - start))

    # get features
    feature_list = get_feature_names(preprocessor)
    assert len(feature_list) == train.shape[1] == test.shape[1]

    if val_df is not None:
        result = train, train_label, test, test_label, val, val_label, feature_list

    else:
        result = train, train_label, test, test_label, feature_list

    return result
//  [python]
def preprocess(train_df, test_df, features, val_df=None, logger=None, objective='classification'):
    """
    The bulk of the preprocessing for a dataset goes here.
    """

    # display datasets
    if logger:
        logger.info('\ntrain_df:\n{}\n{}'.format(train_df.head(5), train_df.shape))
        logger.info('test_df:\n{}\n{}'.format(test_df.head(5), test_df.shape))

    # count number of NAN values per column
    if logger:
        logger.info('')
        for c in train_df.columns:
            logger.info('[TRAIN] {}, no. missing: {:,}'.format(c, train_df[c].isna().sum()))
            logger.info('[TEST] {}, no. missing: {:,}'.format(c, test_df[c].isna().sum()))

    # display column info
    if logger:
        logger.info('\ntrain_df column info:')
        for c in train_df.columns:
            logger.info(f'{c}: {train_df[c].dtype}, {len(train_df[c].unique())}')

    # categorize attributes
    label = features['label']
    numeric = features['numeric']
    categorical = features['categorical']

    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median'))]
    )

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))]
    )

    # time transforms
    start = time.time()

    # perform one-hot encoding for all cat. attributes
    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric),
            ('cat', categorical_transformer, categorical),
        ],
        sparse_threshold=0
    )

    # encode features
    train = preprocessor.fit_transform(train_df)
    test = preprocessor.transform(test_df)
    if val_df is not None:
        val = preprocessor.transform(val_df)

    # encode labels
    if objective == 'classification':
        le = LabelEncoder()
        train_label = le.fit_transform(train_df[label].values.ravel())
        test_label = le.transform(test_df[label].values.ravel())
        if val_df is not None:
            val_label = le.transform(val_df[label].values.ravel())

    # regression
    else:
        assert objective == 'regression'
        train_label = train_df[label].values.ravel()
        test_label = test_df[label].values.ravel()
        if val_df is not None:
            val_label = val_df[label].values.ravel()

    if logger:
        logger.info('transforming features...{:.3f}s'.format(time.time() - start))

    # get features
    feature_list = get_feature_names(preprocessor)
    assert len(feature_list) == train.shape[1] == test.shape[1]

    if val_df is not None:
        result = train, train_label, test, test_label, val, val_label, feature_list

    else:
        result = train, train_label, test, test_label, feature_list

    return result
//  [python]
def test_pipeline_transform_list(self):
        pipeline = Pipeline([("imputer", Imputer()), ("onehotencoder", OneHotEncoder())])
        x = np.ones((3, 2))
        pipeline.fit(x)
        data_mapper = DataMapper([([0, 1], pipeline)])
        result = data_mapper.transform(x)
        assert result.shape == (3, 2)
//  [python]
def test_pipeline_transform_list(self):
        pipeline = Pipeline([("imputer", SimpleImputer()), ("onehotencoder", OneHotEncoder())])
        x = np.ones((3, 2))
        pipeline.fit(x)
        data_mapper = DataMapper([([0, 1], pipeline)])
        result = data_mapper.transform(x)
        assert result.shape == (3, 2)
//  [python]
BASELINE_CLASSIFIER = Pipeline([
    ('imputer', Imputer()),
    ('classifier', LogisticRegression())
])

BASELINE_REGRESSION = Pipeline([
    ('imputer', Imputer()),
    ('classifier', LinearRegression())
])
//  [python]
BASELINE_CLASSIFIER = Pipeline([
    ('imputer', SimpleImputer()),
    ('classifier', LogisticRegression())
])

BASELINE_REGRESSION = Pipeline([
    ('imputer', SimpleImputer()),
    ('classifier', LinearRegression())
])
//  [python]
def test_pipeline_transform_column_transformer(self):
        pipeline = Pipeline([("imputer", Imputer()), ("onehotencoder", OneHotEncoder())])
        x = np.ones((3, 2))
        column_transformer = ColumnTransformer([
            ("column", pipeline, [0, 1])
        ])
        column_transformer.fit(x)
        data_mapper = DataMapper(column_transformer)
        result = data_mapper.transform(x)
        assert result.shape == (3, 2)
//  [python]
def test_pipeline_transform_column_transformer(self):
        pipeline = Pipeline([("imputer", SimpleImputer()), ("onehotencoder", OneHotEncoder())])
        x = np.ones((3, 2))
        column_transformer = ColumnTransformer([
            ("column", pipeline, [0, 1])
        ])
        column_transformer.fit(x)
        data_mapper = DataMapper(column_transformer)
        result = data_mapper.transform(x)
        assert result.shape == (3, 2)
//  [python]
def _impute_em(self, X: np.ndarray) -> np.ndarray:
        """The EM imputation core loop.

        Args:
            X: np.ndarray
                The dataset with missing values.

        Raises:
            RuntimeError: raised if the static checks on the final result fail.

        Returns:
            ndarray: The dataset with imputed values.
        """
        rows, columns = X.shape
        mask = ~np.isnan(X)

        one_to_nc = np.arange(1, columns + 1, step=1)
        missing = one_to_nc * (~mask) - 1
        observed = one_to_nc * mask - 1

        Mu = np.nanmean(X, axis=0)
        observed_rows = np.where(np.isnan(sum(X.T)) is False)[0]
        Sigma = np.cov(
            X[
                observed_rows,
            ].T
        )
        if np.isnan(Sigma).any():
            Sigma = np.diag(np.nanvar(X, axis=0))

        X_reconstructed = X.copy()

        for iteration in range(self.maxit):
            try:
                Mu_new, Sigma_new, X_reconstructed = self._em(
                    X_reconstructed, Mu, Sigma, observed, missing
                )

                if self._converged(Mu, Sigma, Mu_new, Sigma_new):
                    log.debug(f"EM converged after {iteration} iterations.")
                    break

                Mu = Mu_new
                Sigma = Sigma_new
            except BaseException as e:
                log.critical(f"EM step failed. {e}")
                break

        if np.any(np.isnan(X_reconstructed)):
            # fallback to mean imputation in case of singular matrix.
            from sklearn.preprocessing import Imputer
            X_reconstructed = Imputer(strategy="mean").fit_transform(
                X_reconstructed
            )

        return X_reconstructed
//  [python]
def _impute_em(self, X: np.ndarray) -> np.ndarray:
        """The EM imputation core loop.

        Args:
            X: np.ndarray
                The dataset with missing values.

        Raises:
            RuntimeError: raised if the static checks on the final result fail.

        Returns:
            ndarray: The dataset with imputed values.
        """
        rows, columns = X.shape
        mask = ~np.isnan(X)

        one_to_nc = np.arange(1, columns + 1, step=1)
        missing = one_to_nc * (~mask) - 1
        observed = one_to_nc * mask - 1

        Mu = np.nanmean(X, axis=0)
        observed_rows = np.where(np.isnan(sum(X.T)) is False)[0]
        Sigma = np.cov(
            X[
                observed_rows,
            ].T
        )
        if np.isnan(Sigma).any():
            Sigma = np.diag(np.nanvar(X, axis=0))

        X_reconstructed = X.copy()

        for iteration in range(self.maxit):
            try:
                Mu_new, Sigma_new, X_reconstructed = self._em(
                    X_reconstructed, Mu, Sigma, observed, missing
                )

                if self._converged(Mu, Sigma, Mu_new, Sigma_new):
                    log.debug(f"EM converged after {iteration} iterations.")
                    break

                Mu = Mu_new
                Sigma = Sigma_new
            except BaseException as e:
                log.critical(f"EM step failed. {e}")
                break

        if np.any(np.isnan(X_reconstructed)):
            # fallback to mean imputation in case of singular matrix.
            X_reconstructed = SimpleImputer(strategy="mean").fit_transform(
                X_reconstructed
            )

        return X_reconstructed