jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
with wrap.importer():
    from sklearn.preprocessing import Imputer
    from sklearn.linear_model import LogisticRegression

PIPELINE = Imputer(strategy='mean') >> LogisticRegression(max_iter=50, solver='lbfgs')
//  [python]
with wrap.importer():
    from sklearn.impute import SimpleImputer
    from sklearn.linear_model import LogisticRegression

PIPELINE = SimpleImputer(strategy='mean') >> LogisticRegression(max_iter=50, solver='lbfgs')
//  [python]
X = train_data[features]
y = train_data["failure"]
X_test = test_data[features]

# Fill missing values with median for numerical columns
num_cols = X.select_dtypes(exclude="object").columns
imputer = Imputer(strategy="median")
//  [python]
X = train_data[features]
y = train_data["failure"]
X_test = test_data[features]

# Fill missing values with median for numerical columns
num_cols = X.select_dtypes(exclude="object").columns
imputer = SimpleImputer(strategy="median")
//  [python]
numerical_features = [
    "LotFrontage",
    "LotArea",
    "MasVnrArea",
    "BsmtFinSF1",
    "BsmtFinSF2",
    "BsmtUnfSF",
    "TotalBsmtSF",
    "1stFlrSF",
    "2ndFlrSF",
    "LowQualFinSF",
    "GrLivArea",
    "BedroomAbvGr",
    "KitchenAbvGr",
    "TotRmsAbvGrd",
    "Fireplaces",
    "GarageCars",
    "GarageArea",
    "WoodDeckSF",
    "OpenPorchSF",
    "EnclosedPorch",
    "3SsnPorch",
    "ScreenPorch",
    "PoolArea",
    "MiscVal",
    target_name,
]
categorical_features = data.columns.difference(numerical_features)

most_frequent_imputer = Imputer(strategy="most_frequent")
//  [python]
numerical_features = [
    "LotFrontage",
    "LotArea",
    "MasVnrArea",
    "BsmtFinSF1",
    "BsmtFinSF2",
    "BsmtUnfSF",
    "TotalBsmtSF",
    "1stFlrSF",
    "2ndFlrSF",
    "LowQualFinSF",
    "GrLivArea",
    "BedroomAbvGr",
    "KitchenAbvGr",
    "TotRmsAbvGrd",
    "Fireplaces",
    "GarageCars",
    "GarageArea",
    "WoodDeckSF",
    "OpenPorchSF",
    "EnclosedPorch",
    "3SsnPorch",
    "ScreenPorch",
    "PoolArea",
    "MiscVal",
    target_name,
]
categorical_features = data.columns.difference(numerical_features)

most_frequent_imputer = SimpleImputer(strategy="most_frequent")
//  [python]
def _make_cont_enc(df, conts) -> SKLearnEncoder:
    cont_enc = make_pipeline(StandardScaler(), Imputer(fill_value=0)).fit(
        df[conts].values
    )
    return cont_enc
//  [python]
def _make_cont_enc(df, conts) -> SKLearnEncoder:
    cont_enc = make_pipeline(StandardScaler(), SimpleImputer(fill_value=0)).fit(
        df[conts].values
    )
    return cont_enc
//  [python]
def test_simple_impute_pd_na():
    pd = pytest.importorskip("pandas")

    # Impute pandas array of string types.
    df = pd.DataFrame({"feature": pd.Series(["abc", None, "de"], dtype="string")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value="na")
    _assert_array_equal_and_same_dtype(
        imputer.fit_transform(df), np.array([["abc"], ["na"], ["de"]], dtype=object)
    )

    # Impute pandas array of string types without any missing values.
    df = pd.DataFrame({"feature": pd.Series(["abc", "de", "fgh"], dtype="string")})
    imputer = SimpleImputer(fill_value="ok", strategy="constant")
    _assert_array_equal_and_same_dtype(
        imputer.fit_transform(df), np.array([["abc"], ["de"], ["fgh"]], dtype=object)
    )

    # Impute pandas array of integer types.
    df = pd.DataFrame({"feature": pd.Series([1, None, 3], dtype="Int64")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value=-1)
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df), np.array([[1], [-1], [3]], dtype="float64")
    )

    # Use `np.nan` also works.
    imputer = SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=-1)
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df), np.array([[1], [-1], [3]], dtype="float64")
    )

    # Impute pandas array of integer types with 'median' strategy.
    df = pd.DataFrame({"feature": pd.Series([1, None, 2, 3], dtype="Int64")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="median")
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df), np.array([[1], [2], [2], [3]], dtype="float64")
    )

    # Impute pandas array of integer types with 'mean' strategy.
    df = pd.DataFrame({"feature": pd.Series([1, None, 2], dtype="Int64")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="mean")
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df), np.array([[1], [1.5], [2]], dtype="float64")
    )

    # Impute pandas array of float types.
    df = pd.DataFrame({"feature": pd.Series([1.0, None, 3.0], dtype="float64")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value=-2.0)
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df), np.array([[1.0], [-2.0], [3.0]], dtype="float64")
    )

    # Impute pandas array of float types with 'median' strategy.
    df = pd.DataFrame({"feature": pd.Series([1.0, None, 2.0, 3.0], dtype="float64")})
    imputer = SimpleImputer(missing_values=pd.NA, strategy="median")
    _assert_allclose_and_same_dtype(
        imputer.fit_transform(df),
        np.array([[1.0], [2.0], [2.0], [3.0]], dtype="float64"),
    )
//  [python]
def preprocess_adult_x_data(x_train, x_test):
    features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
    categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                            'native-country']

    # prepare data for DT
    def to_float(x):
        return x.astype(float)

    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', Imputer(strategy='constant', fill_value=0)),
               ('to_float', FunctionTransformer(to_float, feature_names_out='one-to-one'))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    preprocessor.fit(x_train)
    encoded_test = preprocessor.transform(x_test)
    return encoded, encoded_test, filter_categorical(preprocessor.get_feature_names_out(), return_feature_names=False)
//  [python]
def preprocess_adult_x_data(x_train, x_test):
    features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
    categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
                            'native-country']

    # prepare data for DT
    def to_float(x):
        return x.astype(float)

    numeric_features = [f for f in features if f not in categorical_features]
    numeric_transformer = Pipeline(
        steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0)),
               ('to_float', FunctionTransformer(to_float, feature_names_out='one-to-one'))]
    )
    categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x_train)
    preprocessor.fit(x_train)
    encoded_test = preprocessor.transform(x_test)
    return encoded, encoded_test, filter_categorical(preprocessor.get_feature_names_out(), return_feature_names=False)
//  [python]
def test_heterogeneous_ensemble_support_missing_values(Ensemble, Estimator, X, y):
    # check that Voting and Stacking predictor delegate the missing values
    # validation to the underlying estimator.
    X = X.copy()
    mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(Imputer(), Estimator())
    ensemble = Ensemble(estimators=[("pipe1", pipe), ("pipe2", pipe)])
    ensemble.fit(X, y).score(X, y)
//  [python]
def test_heterogeneous_ensemble_support_missing_values(Ensemble, Estimator, X, y):
    # check that Voting and Stacking predictor delegate the missing values
    # validation to the underlying estimator.
    X = X.copy()
    mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(SimpleImputer(), Estimator())
    ensemble = Ensemble(estimators=[("pipe1", pipe), ("pipe2", pipe)])
    ensemble.fit(X, y).score(X, y)
//  [python]
def __init__(self, transform_full_config: bool = False):
        self.transform_full_config = transform_full_config
        self.model_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.hp_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.pipeline = make_pipeline(
            NanImputer(),
            make_union(
                make_pipeline(
                    Selector(use={"model"}),
                    self.model_vectorizer,
                ),
                make_pipeline(
                    Selector(ignore={"model"}),
                    self.hp_vectorizer,
                    StandardScaler(),
                    Imputer(strategy="constant", fill_value=0.0),
                ),
            ),
        )
//  [python]
def __init__(self, transform_full_config: bool = False):
        self.transform_full_config = transform_full_config
        self.model_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.hp_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.pipeline = make_pipeline(
            NanImputer(),
            make_union(
                make_pipeline(
                    Selector(use={"model"}),
                    self.model_vectorizer,
                ),
                make_pipeline(
                    Selector(ignore={"model"}),
                    self.hp_vectorizer,
                    StandardScaler(),
                    SimpleImputer(strategy="constant", fill_value=0.0),
                ),
            ),
        )
//  [python]
def setUp(self) -> None:
        cur_dir = Path(self.current_file).parent
        if (cur_dir / "126025.bz2").exists():
            X_train, y_train, X_test, y_test, cat = joblib.load(cur_dir / "126025.bz2")
        else:
            X_train, y_train, X_test, y_test, cat = load_task(126025)
            joblib.dump(
                [X_train, y_train, X_test, y_test, cat],
                cur_dir / "126025.bz2"
            )
        nan_cnt = np.count_nonzero(pd.isna(pd.concat([X_train, X_test])), axis=0)
        cat = np.array(cat)
        cat_na_mask = (nan_cnt > 0) & cat
        num_na_mask = (nan_cnt > 0) & (~cat)
        cat_imputer = Imputer(strategy="constant", fill_value="NA").fit(X_train.loc[:, cat_na_mask])
        # num_imputer = BaseImputer(strategy="median").fit(X_train.loc[:, num_na_mask])
        X_train.loc[:, cat_na_mask] = cat_imputer.transform(X_train.loc[:, cat_na_mask])
        X_test.loc[:, cat_na_mask] = cat_imputer.transform(X_test.loc[:, cat_na_mask])
        # X_train.loc[:, num_na_mask] = num_imputer.transform(X_train.loc[:, num_na_mask])
        # X_test.loc[:, num_na_mask] = num_imputer.transform(X_test.loc[:, num_na_mask])
        ordinal_encoder = OrdinalEncoder(dtype="int").fit(X_train.loc[:, cat])
        transformer = StandardScaler().fit(X_train.loc[:, ~cat])
        X_train.loc[:, cat] = ordinal_encoder.transform(X_train.loc[:, cat])
        X_train.loc[:, ~cat] = transformer.transform(X_train.loc[:, ~cat])
        X_test.loc[:, cat] = ordinal_encoder.transform(X_test.loc[:, cat])
        X_test.loc[:, ~cat] = transformer.transform(X_test.loc[:, ~cat])
        self.cat_indexes = np.arange(len(cat))[cat]
        label_encoder = LabelEncoder().fit(y_train)
        self.y_train = label_encoder.transform(y_train)
        self.y_test = label_encoder.transform(y_test)
        self.X_train = X_train
        self.X_test = X_test
//  [python]
def setUp(self) -> None:
        cur_dir = Path(self.current_file).parent
        if (cur_dir / "126025.bz2").exists():
            X_train, y_train, X_test, y_test, cat = joblib.load(cur_dir / "126025.bz2")
        else:
            X_train, y_train, X_test, y_test, cat = load_task(126025)
            joblib.dump(
                [X_train, y_train, X_test, y_test, cat],
                cur_dir / "126025.bz2"
            )
        nan_cnt = np.count_nonzero(pd.isna(pd.concat([X_train, X_test])), axis=0)
        cat = np.array(cat)
        cat_na_mask = (nan_cnt > 0) & cat
        num_na_mask = (nan_cnt > 0) & (~cat)
        cat_imputer = SimpleImputer(strategy="constant", fill_value="NA").fit(X_train.loc[:, cat_na_mask])
        # num_imputer = BaseImputer(strategy="median").fit(X_train.loc[:, num_na_mask])
        X_train.loc[:, cat_na_mask] = cat_imputer.transform(X_train.loc[:, cat_na_mask])
        X_test.loc[:, cat_na_mask] = cat_imputer.transform(X_test.loc[:, cat_na_mask])
        # X_train.loc[:, num_na_mask] = num_imputer.transform(X_train.loc[:, num_na_mask])
        # X_test.loc[:, num_na_mask] = num_imputer.transform(X_test.loc[:, num_na_mask])
        ordinal_encoder = OrdinalEncoder(dtype="int").fit(X_train.loc[:, cat])
        transformer = StandardScaler().fit(X_train.loc[:, ~cat])
        X_train.loc[:, cat] = ordinal_encoder.transform(X_train.loc[:, cat])
        X_train.loc[:, ~cat] = transformer.transform(X_train.loc[:, ~cat])
        X_test.loc[:, cat] = ordinal_encoder.transform(X_test.loc[:, cat])
        X_test.loc[:, ~cat] = transformer.transform(X_test.loc[:, ~cat])
        self.cat_indexes = np.arange(len(cat))[cat]
        label_encoder = LabelEncoder().fit(y_train)
        self.y_train = label_encoder.transform(y_train)
        self.y_test = label_encoder.transform(y_test)
        self.X_train = X_train
        self.X_test = X_test
//  [python]
def fit(self, X: Dict[str, Any], y: Optional[Any] = None) -> BaseImputer:
        """
        Builds the preprocessor based on the given fit dictionary 'X'.

        Args:
            X (Dict[str, Any]):
                The fit dictionary
            y (Optional[Any]):
                Not Used -- to comply with API

        Returns:
            self:
                returns an instance of self.
        """
        self.check_requirements(X, y)

        # Choose an imputer for any numerical columns
        numerical_columns = X['dataset_properties']['numerical_columns']

        if isinstance(numerical_columns, List) and len(numerical_columns) > 0:
            if self.numerical_strategy == 'constant_zero':
                imputer = sklearn.impute.SimpleImputer(strategy='constant', fill_value=0, copy=False)
                self.preprocessor['numerical'] = imputer
            else:
                imputer = sklearn.impute.SimpleImputer(strategy=self.numerical_strategy, copy=False)
                self.preprocessor['numerical'] = imputer

        return self
//  [python]
def fit(self, X: Dict[str, Any], y: Optional[Any] = None) -> BaseImputer:
        """
        Builds the preprocessor based on the given fit dictionary 'X'.

        Args:
            X (Dict[str, Any]):
                The fit dictionary
            y (Optional[Any]):
                Not Used -- to comply with API

        Returns:
            self:
                returns an instance of self.
        """
        self.check_requirements(X, y)

        # Choose an imputer for any numerical columns
        numerical_columns = X['dataset_properties']['numerical_columns']

        if isinstance(numerical_columns, List) and len(numerical_columns) > 0:
            if self.numerical_strategy == 'constant_zero':
                imputer = SklearnSimpleImputer(strategy='constant', fill_value=0, copy=False)
                self.preprocessor['numerical'] = imputer
            else:
                imputer = SklearnSimpleImputer(strategy=self.numerical_strategy, copy=False)
                self.preprocessor['numerical'] = imputer

        return self
//  [python]
def impute_simple(y_observed, covariates, strategy='mean'):
    N, T, G = y_observed.shape

    # Perform initial imputation on unobserved tissues
    y_imp = Imputer(missing_values=np.nan,
                          strategy=strategy).fit_transform(y_observed.reshape(N, T * G)).reshape(N, T, G)
    return y_imp
//  [python]
def impute_simple(y_observed, covariates, strategy='mean'):
    N, T, G = y_observed.shape

    # Perform initial imputation on unobserved tissues
    y_imp = SimpleImputer(missing_values=np.nan,
                          strategy=strategy).fit_transform(y_observed.reshape(N, T * G)).reshape(N, T, G)
    return y_imp
//  [python]
def create_titanic_pipeline(X_train, y_train):
    def conv(X):
        if isinstance(X, pd.Series):
            return X.values
        return X

    many_to_one_transformer = \
        FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
    many_to_many_transformer = \
        FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1),
                conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
    transformations = ColumnTransformer([
        ("age_fare_1", Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())
        ]), ["age", "fare"]),
        ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
        ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
        ("embarked", Pipeline(steps=[
            ("imputer",
                Imputer(strategy='constant', fill_value='missing')),
            ("encoder", OneHotEncoder(sparse_output=False))]), ["embarked"]),
        ("sex_pclass", OneHotEncoder(sparse_output=False), ["sex", "pclass"])
    ])
    clf = Pipeline(steps=[('preprocessor', transformations),
                          ('classifier',
                           LogisticRegression(solver='lbfgs'))])
    clf.fit(X_train, y_train)
    return clf
//  [python]
def create_titanic_pipeline(X_train, y_train):
    def conv(X):
        if isinstance(X, pd.Series):
            return X.values
        return X

    many_to_one_transformer = \
        FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
    many_to_many_transformer = \
        FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1),
                conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
    transformations = ColumnTransformer([
        ("age_fare_1", Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
        ]), ["age", "fare"]),
        ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
        ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
        ("embarked", Pipeline(steps=[
            ("imputer",
                SimpleImputer(strategy='constant', fill_value='missing')),
            ("encoder", OneHotEncoder(sparse_output=False))]), ["embarked"]),
        ("sex_pclass", OneHotEncoder(sparse_output=False), ["sex", "pclass"])
    ])
    clf = Pipeline(steps=[('preprocessor', transformations),
                          ('classifier',
                           LogisticRegression(solver='lbfgs'))])
    clf.fit(X_train, y_train)
    return clf
//  [python]
def createClassModel(algo_name, catg, nums):
    numeric_transformer = Pipeline(steps=[('imputer', Imputer(strategy='constant', fill_value=0))])

    categorical_transformer = Pipeline(steps=[('imputer', Imputer(strategy='constant', fill_value="MISSING")), ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    preprocesser = ColumnTransformer(transformers=[('num', numeric_transformer, nums), ('cat', categorical_transformer, catg)])

    if algo_name == 'linear_regression':
        #---------------------------------------------
        #setup: Update alpha value
        #---------------------------------------------
        model = Ridge(alpha=100000)  #setup
    elif algo_name == 'random_forest':
        model = RandomForestRegressor()
    else:
        pass
    
    ModelPipeline = Pipeline(steps=[('preprocessor', preprocesser), ("model", model)])
    
    return ModelPipeline
//  [python]
def createClassModel(algo_name, catg, nums):
    numeric_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))])

    categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='constant', fill_value="MISSING")), ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    preprocesser = ColumnTransformer(transformers=[('num', numeric_transformer, nums), ('cat', categorical_transformer, catg)])

    if algo_name == 'linear_regression':
        #---------------------------------------------
        #setup: Update alpha value
        #---------------------------------------------
        model = Ridge(alpha=100000)  #setup
    elif algo_name == 'random_forest':
        model = RandomForestRegressor()
    else:
        pass
    
    ModelPipeline = Pipeline(steps=[('preprocessor', preprocesser), ("model", model)])
    
    return ModelPipeline
//  [python]
print(dataset.isnull().sum().sum())


for i in dataset.columns: # Excluding date and time
    dataset.loc[:, i].astype(float) # Defining the format

values = dataset.values.astype(float) # create object for dataframe values

dataset['General_Power'] = np.sqrt(np.square(values[:,0]) + np.square(values[:,1]))


# Make imputting for data that change all NaN values to most frequent 
ImputedModule = Imputer(missing_values= nan, strategy ='most_frequent')
//  [python]
print(dataset.isnull().sum().sum())


for i in dataset.columns: # Excluding date and time
    dataset.loc[:, i].astype(float) # Defining the format

values = dataset.values.astype(float) # create object for dataframe values

dataset['General_Power'] = np.sqrt(np.square(values[:,0]) + np.square(values[:,1]))


# Make imputting for data that change all NaN values to most frequent 
ImputedModule = SimpleImputer(missing_values= nan, strategy ='most_frequent')
//  [python]
def get_generic_preprocessing_pipeline() -> Pipeline:
    cat_pipeline = Pipeline(
        [
            ("imputer", Imputer(strategy="most_frequent")),
            ("one-hot", OneHotEncoder(sparse=False, handle_unknown="ignore")),
        ]
    )

    num_pipeline = Pipeline(
        [
            ("imputer", Imputer(strategy="mean")),
            ("scaler", StandardScaler()),
        ]
    )

    pipeline = Pipeline(
        [
            (
                "transformers",
                make_column_transformer(
                    (cat_pipeline, make_column_selector(dtype_include=object)),
                    (num_pipeline, make_column_selector(dtype_include=np.number)),
                ),
            )
        ]
    )

    return pipeline
//  [python]
def get_generic_preprocessing_pipeline() -> Pipeline:
    cat_pipeline = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="most_frequent")),
            ("one-hot", OneHotEncoder(sparse=False, handle_unknown="ignore")),
        ]
    )

    num_pipeline = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean")),
            ("scaler", StandardScaler()),
        ]
    )

    pipeline = Pipeline(
        [
            (
                "transformers",
                make_column_transformer(
                    (cat_pipeline, make_column_selector(dtype_include=object)),
                    (num_pipeline, make_column_selector(dtype_include=np.number)),
                ),
            )
        ]
    )

    return pipeline
//  [python]
def _setup_transformers(self):
        if isinstance(self.scaler, TransformerMixin):
            scaler = self.scaler
        elif self.scaler == "standard":
            scaler = StandardScaler()
        elif self.scaler == "minmax":
            scaler = MinMaxScaler()
        else:
            scaler = RobustScaler()

        target_is_multilabel = self.target_info["type_"] in [
            "multilabel-indicator",
            "multiclass-multioutput",
            "continuous-multioutput",
        ]
        if isinstance(self.high_cardinality_encoder, TransformerMixin):
            high_cardinality_encoder = self.high_cardinality_encoder
        elif self.high_cardinality_encoder == "target":
            if target_is_multilabel:
                warnings.warn(
                    "TargetEncoder is not supported for multilabel or multioutput targets. "
                    "Switching to OrdinalEncoder.",
                    stacklevel=2,
                )
                high_cardinality_encoder = OrdinalEncoder(
                    handle_unknown="use_encoded_value", unknown_value=99999
                )
            else:
                high_cardinality_encoder = TargetEncoder(
                    task=self.task, handle_unknown="ignore"
                )
        else:
            high_cardinality_encoder = OrdinalEncoder(
                handle_unknown="use_encoded_value", unknown_value=99999
            )

        cat_date_imputer = Imputer(strategy="most_frequent")

        if isinstance(self.numeric_imputer, TransformerMixin):
            num_imputer = self.numeric_imputer
        elif self.numeric_imputer == "iterative":
            from sklearn.experimental import enable_iterative_imputer
            from sklearn.impute import IterativeImputer

            num_imputer = IterativeImputer(random_state=self.random_state)
        else:
            num_imputer = Imputer(strategy="mean")

        numeric_preprocessor = Pipeline(
            [("numeric_imputer", num_imputer), ("scaler", scaler)]
        )
        cat_low_preprocessor = Pipeline(
            [
                ("categorical_imputer", cat_date_imputer),
                (
                    "one-hot_encoder",
                    OneHotEncoder(
                        drop="if_binary", handle_unknown="ignore", sparse=False
                    ),
                ),
            ]
        )
        cat_high_preprocessor = Pipeline(
            [
                ("categorical_imputer", cat_date_imputer),
                (
                    "high_cardinality_encoder",
                    high_cardinality_encoder,
                ),
            ],
        )
        datetime_preprocessor = Pipeline(
            [
                (
                    "datetime_encoder",
                    DatetimeEncoder(),
                ),
                ("datetime_imputer", cat_date_imputer),
            ],
        )
        return (
            numeric_preprocessor,
            cat_low_preprocessor,
            cat_high_preprocessor,
            datetime_preprocessor,
        )
//  [python]
def _setup_transformers(self):
        if isinstance(self.scaler, TransformerMixin):
            scaler = self.scaler
        elif self.scaler == "standard":
            scaler = StandardScaler()
        elif self.scaler == "minmax":
            scaler = MinMaxScaler()
        else:
            scaler = RobustScaler()

        target_is_multilabel = self.target_info["type_"] in [
            "multilabel-indicator",
            "multiclass-multioutput",
            "continuous-multioutput",
        ]
        if isinstance(self.high_cardinality_encoder, TransformerMixin):
            high_cardinality_encoder = self.high_cardinality_encoder
        elif self.high_cardinality_encoder == "target":
            if target_is_multilabel:
                warnings.warn(
                    "TargetEncoder is not supported for multilabel or multioutput targets. "
                    "Switching to OrdinalEncoder.",
                    stacklevel=2,
                )
                high_cardinality_encoder = OrdinalEncoder(
                    handle_unknown="use_encoded_value", unknown_value=99999
                )
            else:
                high_cardinality_encoder = TargetEncoder(
                    task=self.task, handle_unknown="ignore"
                )
        else:
            high_cardinality_encoder = OrdinalEncoder(
                handle_unknown="use_encoded_value", unknown_value=99999
            )

        cat_date_imputer = SimpleImputer(strategy="most_frequent")

        if isinstance(self.numeric_imputer, TransformerMixin):
            num_imputer = self.numeric_imputer
        elif self.numeric_imputer == "iterative":
            from sklearn.experimental import enable_iterative_imputer
            from sklearn.impute import IterativeImputer

            num_imputer = IterativeImputer(random_state=self.random_state)
        else:
            num_imputer = SimpleImputer(strategy="mean")

        numeric_preprocessor = Pipeline(
            [("numeric_imputer", num_imputer), ("scaler", scaler)]
        )
        cat_low_preprocessor = Pipeline(
            [
                ("categorical_imputer", cat_date_imputer),
                (
                    "one-hot_encoder",
                    OneHotEncoder(
                        drop="if_binary", handle_unknown="ignore", sparse=False
                    ),
                ),
            ]
        )
        cat_high_preprocessor = Pipeline(
            [
                ("categorical_imputer", cat_date_imputer),
                (
                    "high_cardinality_encoder",
                    high_cardinality_encoder,
                ),
            ],
        )
        datetime_preprocessor = Pipeline(
            [
                (
                    "datetime_encoder",
                    DatetimeEncoder(),
                ),
                ("datetime_imputer", cat_date_imputer),
            ],
        )
        return (
            numeric_preprocessor,
            cat_low_preprocessor,
            cat_high_preprocessor,
            datetime_preprocessor,
        )
//  [python]
train_data = train_data.replace('Medium', "Low")
test_data = test_data.replace('Medium', "Low")

train_labels = label_binarize(train_data['score_text'], classes=['High', 'Low'])
test_labels = label_binarize(test_data['score_text'], classes=['High', 'Low'])

impute1_and_onehot = Pipeline([('imputer1', Imputer(strategy='most_frequent')),
                               ('onehot', OneHotEncoder(handle_unknown='ignore'))])
//  [python]
train_data = train_data.replace('Medium', "Low")
test_data = test_data.replace('Medium', "Low")

train_labels = label_binarize(train_data['score_text'], classes=['High', 'Low'])
test_labels = label_binarize(test_data['score_text'], classes=['High', 'Low'])

impute1_and_onehot = Pipeline([('imputer1', SimpleImputer(strategy='most_frequent')),
                               ('onehot', OneHotEncoder(handle_unknown='ignore'))])
//  [python]
def test_pipeline_missing_values_leniency():
    # check that pipeline let the missing values validation to
    # the underlying transformers and predictors.
    X, y = iris.data, iris.target
    mask = np.random.choice([1, 0], X.shape, p=[.1, .9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(Imputer(), LogisticRegression())
    assert pipe.fit(X, y).score(X, y) > 0.4
//  [python]
def test_pipeline_missing_values_leniency():
    # check that pipeline let the missing values validation to
    # the underlying transformers and predictors.
    X, y = iris.data, iris.target
    mask = np.random.choice([1, 0], X.shape, p=[.1, .9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(SimpleImputer(), LogisticRegression())
    assert pipe.fit(X, y).score(X, y) > 0.4
//  [python]
def test_simple_imputer_no_nan_inputs(self):
        model = Imputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        for backend in ["torch", "torch.jit"]:
            self._test_simple_imputer(model, data, backend)
//  [python]
def test_simple_imputer_no_nan_inputs(self):
        model = SimpleImputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        for backend in ["torch", "torch.jit"]:
            self._test_simple_imputer(model, data, backend)
//  [python]
def initZgU(
        self,
        pmean=0,
        pvar=1.0,
        qmean="random",
        qvar=1.0,
        qE=None,
        qE2=None,
        Y=None,
        impute=True,
        idx_inducing=None,
        weight_views=False,
    ):
        assert (
            idx_inducing is not None
        ), "Stop: ZgU nodes is used without inducing points"

        ## Initialise prior distribution (P)

        # mean
        pmean = np.ones((self.N, self.K)) * pmean

        ## Initialise variational distribution (Q)

        # variance
        qvar = np.ones((self.N, self.K)) * qvar

        # mean
        if qmean is not None:
            if isinstance(qmean, str):
                # Random initialisation
                if qmean == "random":
                    qmean = stats.norm.rvs(loc=0, scale=1, size=(self.N, self.K))

                # Random initialisation with orthogonal factors
                elif qmean == "orthogonal":
                    pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                    pca.fit(stats.norm.rvs(loc=0, scale=1, size=(self.N, 9999)).T)
                    qmean = pca.components_.T

                # PCA initialisation
                elif qmean == "pca":
                    # whiten=True scales the principal components to match the prior N(0,1)
                    pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                    Ytmp = np.concatenate(Y, axis=1)

                    if impute == True:
                        if np.any(np.isnan(Ytmp)):
                            imp = Imputer(missing_values=np.NaN, strategy="mean")
                            imp.fit(Ytmp)
                            Ytmp = imp.transform(Ytmp)

                    pca.fit(Ytmp)
                    qmean = pca.transform(Ytmp)

                # scale factor values from -1 to 1 (per factor)
                qmean = (
                    2.0 * (qmean - np.min(qmean, axis=0)) / np.ptp(qmean, axis=0) - 1
                )

            elif isinstance(qmean, np.ndarray):
                assert qmean.shape == (
                    self.N,
                    self.K,
                ), "Wrong shape for the expectation of the Q distribution of Z"

            elif isinstance(qmean, (int, float)):
                qmean = np.ones((self.N, self.K)) * qmean

            else:
                print("Wrong initialisation for Z")
                exit()

        self.nodes["Z"] = ZgU_node(
            dim=(self.N, self.K),
            pmean=pmean,
            pvar=pvar,
            qmean=qmean,
            qvar=qvar,
            qE=qE,
            qE2=qE2,
            idx_inducing=idx_inducing,
            weight_views=weight_views,
        )
//  [python]
def initZgU(
        self,
        pmean=0,
        pvar=1.0,
        qmean="random",
        qvar=1.0,
        qE=None,
        qE2=None,
        Y=None,
        impute=True,
        idx_inducing=None,
        weight_views=False,
    ):
        assert (
            idx_inducing is not None
        ), "Stop: ZgU nodes is used without inducing points"

        ## Initialise prior distribution (P)

        # mean
        pmean = np.ones((self.N, self.K)) * pmean

        ## Initialise variational distribution (Q)

        # variance
        qvar = np.ones((self.N, self.K)) * qvar

        # mean
        if qmean is not None:
            if isinstance(qmean, str):
                # Random initialisation
                if qmean == "random":
                    qmean = stats.norm.rvs(loc=0, scale=1, size=(self.N, self.K))

                # Random initialisation with orthogonal factors
                elif qmean == "orthogonal":
                    pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                    pca.fit(stats.norm.rvs(loc=0, scale=1, size=(self.N, 9999)).T)
                    qmean = pca.components_.T

                # PCA initialisation
                elif qmean == "pca":
                    # whiten=True scales the principal components to match the prior N(0,1)
                    pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                    Ytmp = np.concatenate(Y, axis=1)

                    if impute == True:
                        if np.any(np.isnan(Ytmp)):
                            imp = SimpleImputer(missing_values=np.NaN, strategy="mean")
                            imp.fit(Ytmp)
                            Ytmp = imp.transform(Ytmp)

                    pca.fit(Ytmp)
                    qmean = pca.transform(Ytmp)

                # scale factor values from -1 to 1 (per factor)
                qmean = (
                    2.0 * (qmean - np.min(qmean, axis=0)) / np.ptp(qmean, axis=0) - 1
                )

            elif isinstance(qmean, np.ndarray):
                assert qmean.shape == (
                    self.N,
                    self.K,
                ), "Wrong shape for the expectation of the Q distribution of Z"

            elif isinstance(qmean, (int, float)):
                qmean = np.ones((self.N, self.K)) * qmean

            else:
                print("Wrong initialisation for Z")
                exit()

        self.nodes["Z"] = ZgU_node(
            dim=(self.N, self.K),
            pmean=pmean,
            pvar=pvar,
            qmean=qmean,
            qvar=qvar,
            qE=qE,
            qE2=qE2,
            idx_inducing=idx_inducing,
            weight_views=weight_views,
        )
//  [python]
def test_simple_imputer_nan_to_0(self):
        model = Imputer(strategy="constant", fill_value=0)
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        for backend in ["torch", "torch.jit"]:
            self._test_simple_imputer(model, data, backend)
//  [python]
def test_simple_imputer_nan_to_0(self):
        model = SimpleImputer(strategy="constant", fill_value=0)
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        for backend in ["torch", "torch.jit"]:
            self._test_simple_imputer(model, data, backend)
//  [python]
def test_simple_imputer_float_inputs_tvm(self):
        model = Imputer(strategy="mean", fill_value="nan")
        data = np.array([[1, 2], [np.nan, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def test_simple_imputer_float_inputs_tvm(self):
        model = SimpleImputer(strategy="mean", fill_value="nan")
        data = np.array([[1, 2], [np.nan, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def test_simple_imputer_no_nan_inputs_tvm(self):
        model = Imputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def test_simple_imputer_no_nan_inputs_tvm(self):
        model = SimpleImputer(missing_values=0, strategy="most_frequent")
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def initSZ(
        self,
        pmean_T0=0.0,
        pmean_T1=0.0,
        pvar_T0=1.0,
        pvar_T1=1.0,
        ptheta=1.0,
        qmean_T0=0.0,
        qmean_T1="random",
        qvar_T0=1.0,
        qvar_T1=1.0,
        qtheta=1.0,
        qEZ_T0=None,
        qEZ_T1=None,
        qET=None,
        Y=None,
        impute=False,
        weight_views=False,
    ):
        """Method to initialise sparse factors with a spike and slab prior

        PARAMETERS
        ----------
        (...)
        """

        # TODO:
        # - add different ways to initialise the expectations of the posterior: random, orthogonal, PCA

        ## Initialise prior distribution (P)

        ## Initialise variational distribution (Q)
        if isinstance(qmean_T1, str):
            if qmean_T1 == "random":
                qmean_T1 = stats.norm.rvs(loc=0, scale=1, size=(self.N, self.K))
            elif qmean_T1 == "pca":
                pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                Ytmp = np.concatenate(Y, axis=1)

                if impute == True:
                    if np.any(np.isnan(Ytmp)):
                        imp = Imputer(missing_values=np.NaN, strategy="mean")
                        imp.fit(Ytmp)
                        Ytmp = imp.transform(Ytmp)

                pca.fit(Ytmp)
                qmean_T1 = pca.transform(Ytmp)
            else:
                print("%s initialisation not implemented for Z" % qmean_T1)
                exit()

        elif isinstance(qmean_T1, np.ndarray):
            assert qmean_T1.shape == (self.N, self.K), "Wrong dimensionality"

        elif isinstance(qmean_T1, (int, float)):
            qmean_T1 *= np.ones((self.N, self.K))

        else:
            print("Wrong initialisation for Z")
            exit(1)

        self.nodes["Z"] = SZ_Node(
            dim=(self.N, self.K),
            ptheta=ptheta,
            pmean_T0=pmean_T0,
            pvar_T0=pvar_T0,
            pmean_T1=pmean_T1,
            pvar_T1=pvar_T1,
            qtheta=qtheta,
            qmean_T0=qmean_T0,
            qvar_T0=qvar_T0,
            qmean_T1=qmean_T1,
            qvar_T1=qvar_T1,
            qET=qET,
            qEZ_T0=qEZ_T0,
            qEZ_T1=qEZ_T1,
            weight_views=weight_views,
        )
//  [python]
def initSZ(
        self,
        pmean_T0=0.0,
        pmean_T1=0.0,
        pvar_T0=1.0,
        pvar_T1=1.0,
        ptheta=1.0,
        qmean_T0=0.0,
        qmean_T1="random",
        qvar_T0=1.0,
        qvar_T1=1.0,
        qtheta=1.0,
        qEZ_T0=None,
        qEZ_T1=None,
        qET=None,
        Y=None,
        impute=False,
        weight_views=False,
    ):
        """Method to initialise sparse factors with a spike and slab prior

        PARAMETERS
        ----------
        (...)
        """

        # TODO:
        # - add different ways to initialise the expectations of the posterior: random, orthogonal, PCA

        ## Initialise prior distribution (P)

        ## Initialise variational distribution (Q)
        if isinstance(qmean_T1, str):
            if qmean_T1 == "random":
                qmean_T1 = stats.norm.rvs(loc=0, scale=1, size=(self.N, self.K))
            elif qmean_T1 == "pca":
                pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
                Ytmp = np.concatenate(Y, axis=1)

                if impute == True:
                    if np.any(np.isnan(Ytmp)):
                        imp = SimpleImputer(missing_values=np.NaN, strategy="mean")
                        imp.fit(Ytmp)
                        Ytmp = imp.transform(Ytmp)

                pca.fit(Ytmp)
                qmean_T1 = pca.transform(Ytmp)
            else:
                print("%s initialisation not implemented for Z" % qmean_T1)
                exit()

        elif isinstance(qmean_T1, np.ndarray):
            assert qmean_T1.shape == (self.N, self.K), "Wrong dimensionality"

        elif isinstance(qmean_T1, (int, float)):
            qmean_T1 *= np.ones((self.N, self.K))

        else:
            print("Wrong initialisation for Z")
            exit(1)

        self.nodes["Z"] = SZ_Node(
            dim=(self.N, self.K),
            ptheta=ptheta,
            pmean_T0=pmean_T0,
            pvar_T0=pvar_T0,
            pmean_T1=pmean_T1,
            pvar_T1=pvar_T1,
            qtheta=qtheta,
            qmean_T0=qmean_T0,
            qvar_T0=qvar_T0,
            qmean_T1=qmean_T1,
            qvar_T1=qvar_T1,
            qET=qET,
            qEZ_T0=qEZ_T0,
            qEZ_T1=qEZ_T1,
            weight_views=weight_views,
        )
//  [python]
def test_simple_imputer_nan_to_0_tvm(self):
        model = Imputer(strategy="constant", fill_value=0)
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def test_simple_imputer_nan_to_0_tvm(self):
        model = SimpleImputer(strategy="constant", fill_value=0)
        data = np.array([[1, 2], [1, 3], [7, 6]], dtype=np.float32)

        self._test_simple_imputer(model, data, "tvm")
//  [python]
def __init__(self, column_types, label_column, tokenizer_name, label_generator=None, cfg=None):
        self._column_types = column_types
        self._label_column = label_column
        cfg = base_preprocess_cfg().clone_merge(cfg)
        self._cfg = cfg
        self._feature_generators = dict()
        self._label_generator = label_generator
        self._label_scaler = StandardScaler()   # Scaler used for numerical labels
        for col_name, col_type in self._column_types.items():
            if col_name == self._label_column:
                continue
            if col_type == _C.TEXT:
                continue
            elif col_type == _C.CATEGORICAL:
                generator = CategoryFeatureGenerator(
                    cat_order='count',
                    minimum_cat_count=cfg.categorical.minimum_cat_count,
                    maximum_num_cat=cfg.categorical.maximum_num_cat,
                    verbosity=0)
                self._feature_generators[col_name] = generator
            elif col_type == _C.NUMERICAL:
                generator = Pipeline(
                    [('imputer', Imputer()),
                     ('scaler', StandardScaler(with_mean=cfg.numerical.scaler_with_mean,
                                               with_std=cfg.numerical.scaler_with_std))]
                )
                self._feature_generators[col_name] = generator

        self._tokenizer_name = tokenizer_name
        self._tokenizer = get_tokenizer(tokenizer_name)
        self._fit_called = False

        # Some columns will be ignored
        self._ignore_columns_set = set()
        self._text_feature_names = []
        self._categorical_feature_names = []
        self._categorical_num_categories = []
        self._numerical_feature_names = []
//  [python]
def __init__(self, column_types, label_column, tokenizer_name, label_generator=None, cfg=None):
        self._column_types = column_types
        self._label_column = label_column
        cfg = base_preprocess_cfg().clone_merge(cfg)
        self._cfg = cfg
        self._feature_generators = dict()
        self._label_generator = label_generator
        self._label_scaler = StandardScaler()   # Scaler used for numerical labels
        for col_name, col_type in self._column_types.items():
            if col_name == self._label_column:
                continue
            if col_type == _C.TEXT:
                continue
            elif col_type == _C.CATEGORICAL:
                generator = CategoryFeatureGenerator(
                    cat_order='count',
                    minimum_cat_count=cfg.categorical.minimum_cat_count,
                    maximum_num_cat=cfg.categorical.maximum_num_cat,
                    verbosity=0)
                self._feature_generators[col_name] = generator
            elif col_type == _C.NUMERICAL:
                generator = Pipeline(
                    [('imputer', SimpleImputer()),
                     ('scaler', StandardScaler(with_mean=cfg.numerical.scaler_with_mean,
                                               with_std=cfg.numerical.scaler_with_std))]
                )
                self._feature_generators[col_name] = generator

        self._tokenizer_name = tokenizer_name
        self._tokenizer = get_tokenizer(tokenizer_name)
        self._fit_called = False

        # Some columns will be ignored
        self._ignore_columns_set = set()
        self._text_feature_names = []
        self._categorical_feature_names = []
        self._categorical_num_categories = []
        self._numerical_feature_names = []
//  [python]
def get_impute_zero_score(X_missing, y_missing):
    imputer = Imputer(
        missing_values=np.nan, strategy="constant", fill_value=0
    )
    zero_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing)
    return zero_impute_scores.mean(), zero_impute_scores.std()
//  [python]
def get_impute_zero_score(X_missing, y_missing):
    imputer = SimpleImputer(
        missing_values=np.nan, add_indicator=True, strategy="constant", fill_value=0
    )
    zero_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing)
    return zero_impute_scores.mean(), zero_impute_scores.std()