jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def get_data_transformation(self):
        
        try:
            logging.info('Data Transformation initiated')
            
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']
            
            logging.info('Pipeline Initiated')
            
            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='median')),
                ('scaler',StandardScaler())

                ]

            )
            
            # Categorigal Pipeline
            cat_pipeline=Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='most_frequent')),
                ('ordinalencoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]

            )
            
            preprocessor=ColumnTransformer([
            ('num_pipeline',num_pipeline,numerical_cols),
            ('cat_pipeline',cat_pipeline,categorical_cols)
            ])
            
            return preprocessor
            

            
            
        
        except Exception as e:
            logging.info("Exception occured in the initiate_datatransformation")

            raise customexception(e,sys)
//  [python]
def test_1(self):
        """
        From https://scikit-learn.org/1.1/modules/generated/sklearn.impute.SimpleImputer.html
        >>> import numpy as np
        >>> from sklearn.impute import SimpleImputer
        >>> imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
        >>> imp_mean.fit([[7, 2, 3], [4, np.nan, 6], [10, 5, 9]])
        SimpleImputer()
        >>> X = [[np.nan, 2, 3], [4, np.nan, 6], [10, np.nan, 9]]
        >>> print(imp_mean.transform(X))
        [[ 7.   2.   3. ]
        [ 4.   3.5  6. ]
        [10.   3.5  9. ]]
        """
        sk_imp_mean = SkSimpleImputer(missing_values=np.nan, strategy="mean")
        training = [[7, 2, 3], [4, np.nan, 6], [10, 5, 9]]
        sk_imp_mean.fit(training)

        X = [[np.nan, 2, 3], [4, np.nan, 6], [10, np.nan, 9]]
        sk_transformed = sk_imp_mean.transform(X)
        for target in self.targets:
            training = Convert(astype=target).transform(training)
            X = Convert(astype=target).transform(X)
            rasl_imp_mean = RaslSimpleImputer(missing_values=np.nan, strategy="mean")
            rasl_imp_mean.fit(training)
            rasl_transformed = rasl_imp_mean.transform(X)
            _check_trained_simple_imputer(self, sk_imp_mean, rasl_imp_mean, target)
            _check_data(self, sk_transformed, rasl_transformed, target)
//  [python]
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", Imputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)

# For categoricals, we:
# 1. Convert boolean values to strings
# 2. Impute missing values with the string "missing"
# 3. One hot encode the data (ignoring new categorical levels at prediction time)
# You can set `handle_unknown='error'` to make your model raise an error at prediction time if
# it encounters a new categorical level
categorical_pipeline = Pipeline(
    steps=[
        ("bool_to_string", FunctionTransformer(to_string)),
        ("imputer", Imputer(strategy="constant", fill_value="missing")),
        ("onehot", OneHotEncoder(handle_unknown="ignore")),
    ]
)
//  [python]
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)

# For categoricals, we:
# 1. Convert boolean values to strings
# 2. Impute missing values with the string "missing"
# 3. One hot encode the data (ignoring new categorical levels at prediction time)
# You can set `handle_unknown='error'` to make your model raise an error at prediction time if
# it encounters a new categorical level
categorical_pipeline = Pipeline(
    steps=[
        ("bool_to_string", FunctionTransformer(to_string)),
        ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
        ("onehot", OneHotEncoder(handle_unknown="ignore")),
    ]
)
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPRegressor(hidden_layer_sizes=(150, 100, 50),
                                                          max_iter=500, random_state=0))])

        clf.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, verbose=False)
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns='survived')
        self.y = data.survived

        numeric_features = ['age', 'fare', 'sibsp', 'parch']
        numeric_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())])

        categorical_features = ['gender', 'class', 'embarked']
        categorical_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)])

        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', MLPRegressor(hidden_layer_sizes=(150, 100, 50),
                                                          max_iter=500, random_state=0))])

        clf.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, verbose=False)
//  [python]
def preprocess(rain):
    # Drop unnecessary columns
    cols_to_drop = ["Date", "Location", "RainTomorrow", "Rainfall"]
    rain.drop(cols_to_drop, axis=1, inplace=True)

    # Compute missing proportions
    missing_props = rain.isna().mean(axis=0)
    # Drop cols with >40% missing
    over_threshold = missing_props[missing_props >= 0.4]
    rain.drop(over_threshold.index, axis=1, inplace=True)

    # Build feature and target arrays
    X = rain.drop("RainToday", axis=1)
    y = rain.RainToday

    # Build the pipeline for categorical features
    categorical_pipeline = Pipeline(
        steps=[
            ("impute", Imputer(strategy="most_frequent")),
            ("oh-encode", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    # Build the pipeline for numerical features
    numeric_pipeline = Pipeline(
        steps=[
            ("impute", Imputer(strategy="mean")),
            ("scale", StandardScaler())
        ]
    )

    # Extract numeric and categorical col names
    cat_cols = X.select_dtypes(exclude="number").columns
    num_cols = X.select_dtypes(include="number").columns

    # Build the full transformer combining the above pipelines
    full_processor = ColumnTransformer(
        transformers=[
            ("numeric", numeric_pipeline, num_cols),
            ("categorical", categorical_pipeline, cat_cols),
        ]
    )

    # Apply preprocessing
    X_processed = full_processor.fit_transform(X)
    y_processed = Imputer(strategy="most_frequent").fit_transform(
        y.values.reshape(-1, 1)
    )

    return X_processed, y_processed
//  [python]
def preprocess(rain):
    # Drop unnecessary columns
    cols_to_drop = ["Date", "Location", "RainTomorrow", "Rainfall"]
    rain.drop(cols_to_drop, axis=1, inplace=True)

    # Compute missing proportions
    missing_props = rain.isna().mean(axis=0)
    # Drop cols with >40% missing
    over_threshold = missing_props[missing_props >= 0.4]
    rain.drop(over_threshold.index, axis=1, inplace=True)

    # Build feature and target arrays
    X = rain.drop("RainToday", axis=1)
    y = rain.RainToday

    # Build the pipeline for categorical features
    categorical_pipeline = Pipeline(
        steps=[
            ("impute", SimpleImputer(strategy="most_frequent")),
            ("oh-encode", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    # Build the pipeline for numerical features
    numeric_pipeline = Pipeline(
        steps=[
            ("impute", SimpleImputer(strategy="mean")),
            ("scale", StandardScaler())
        ]
    )

    # Extract numeric and categorical col names
    cat_cols = X.select_dtypes(exclude="number").columns
    num_cols = X.select_dtypes(include="number").columns

    # Build the full transformer combining the above pipelines
    full_processor = ColumnTransformer(
        transformers=[
            ("numeric", numeric_pipeline, num_cols),
            ("categorical", categorical_pipeline, cat_cols),
        ]
    )

    # Apply preprocessing
    X_processed = full_processor.fit_transform(X)
    y_processed = SimpleImputer(strategy="most_frequent").fit_transform(
        y.values.reshape(-1, 1)
    )

    return X_processed, y_processed
//  [python]
def fillMissingData(X):
    print("Preenchendo dados que estão faltando...")
    from sklearn.preprocessing import Imputer
    imputer = Imputer(missing_values=np.nan, strategy='median')
    X[:,1:] = imputer.fit_transform(X[:,1:])
    print("ok!")
    return X
//  [python]
def fillMissingData(X):
    print("Preenchendo dados que estão faltando...")
    from sklearn.impute import SimpleImputer
    imputer = SimpleImputer(missing_values=np.nan, strategy='median')
    X[:,1:] = imputer.fit_transform(X[:,1:])
    print("ok!")
    return X
//  [python]
def test_pipeline_missing_values_leniency():
    # check that pipeline let the missing values validation to
    # the underlying transformers and predictors.
    X, y = iris.data, iris.target
    mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(Imputer(), LogisticRegression())
    assert pipe.fit(X, y).score(X, y) > 0.4
//  [python]
def test_pipeline_missing_values_leniency():
    # check that pipeline let the missing values validation to
    # the underlying transformers and predictors.
    X, y = iris.data, iris.target
    mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool)
    X[mask] = np.nan
    pipe = make_pipeline(SimpleImputer(), LogisticRegression())
    assert pipe.fit(X, y).score(X, y) > 0.4
//  [python]
def impute_numerical(numerical: pd.DataFrame, strategy: str = "median", **kwargs) -> pd.DataFrame:
    """Imputes numerical features with the given strategy

    Parameters:
        numerical (pd.DataFrame): DataFrame with numerical features
        strategy (str): The imputation strategy. Default is 'median'

    Returns:
        pd.DataFrame: DataFrame with imputed numerical features
    """
    if strategy not in ["median", "mean"]:
        raise ValueError(f"Invalid strategy for numerical: {strategy}")

    if numerical.isnull().sum().sum() == 0:
        return numerical

    imputer = Imputer(strategy=strategy, **kwargs)
    numerical_imputed = pd.DataFrame(imputer.fit_transform(numerical), columns=numerical.columns)

    return numerical_imputed
//  [python]
def impute_numerical(numerical: pd.DataFrame, strategy: str = "median", **kwargs) -> pd.DataFrame:
    """Imputes numerical features with the given strategy

    Parameters:
        numerical (pd.DataFrame): DataFrame with numerical features
        strategy (str): The imputation strategy. Default is 'median'

    Returns:
        pd.DataFrame: DataFrame with imputed numerical features
    """
    if strategy not in ["median", "mean"]:
        raise ValueError(f"Invalid strategy for numerical: {strategy}")

    if numerical.isnull().sum().sum() == 0:
        return numerical

    imputer = SimpleImputer(strategy=strategy, **kwargs)
    numerical_imputed = pd.DataFrame(imputer.fit_transform(numerical), columns=numerical.columns)

    return numerical_imputed
//  [python]
def sklearn_pipeline(train_data):
    from sklearn.compose import ColumnTransformer
    from sklearn.ensemble import GradientBoostingClassifier
    from sklearn.preprocessing import Imputer
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import StandardScaler

    X, y = train_data

    numeric_transformer = Pipeline(
        [("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
    )

    preprocessor = ColumnTransformer([("num", numeric_transformer, X.columns)])

    pipe = Pipeline(
        [("preprocess", preprocessor), ("classifier", GradientBoostingClassifier())]
    )

    pipe.fit(X, y)

    return pipe
//  [python]
def sklearn_pipeline(train_data):
    from sklearn.compose import ColumnTransformer
    from sklearn.ensemble import GradientBoostingClassifier
    from sklearn.impute import SimpleImputer
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import StandardScaler

    X, y = train_data

    numeric_transformer = Pipeline(
        [("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
    )

    preprocessor = ColumnTransformer([("num", numeric_transformer, X.columns)])

    pipe = Pipeline(
        [("preprocess", preprocessor), ("classifier", GradientBoostingClassifier())]
    )

    pipe.fit(X, y)

    return pipe
//  [python]
def load_mimic(base_dir: str = './data/'):
    data = pd.read_csv(f'{base_dir}/mimic-ii/full_cohort_data.csv')
    # data.drop('hgb_first')
    fs = [
        'aline_flg',
        'gender_num',
        # 'hosp_exp_flg',
        # 'icu_exp_flg',
        # 'day_28_flg',
        # 'censor_flg',
        'sepsis_flg', 'chf_flg', 'afib_flg',
        'renal_flg', 'liver_flg', 'copd_flg', 'cad_flg', 'stroke_flg',
        'mal_flg', 'resp_flg',
    ]
    features = fs
    data1 = data[fs].values
    imp_mean = Imputer(missing_values=np.nan, strategy='mean')
    data1 = imp_mean.fit_transform(data1)

    f2 = fs.copy()
    f2.append('day_icu_intime')
    f2.append('service_unit')
    f2.append('day_28_flg')
    f2.append('hospital_los_day')
    f2.append('icu_exp_flg')
    f2.append('hosp_exp_flg')
    f2.append('censor_flg')
    f2.append('mort_day_censored')
    f2 = data.columns.difference(f2)
    data2 = data[f2].values
    imp_mean = Imputer(missing_values=np.nan, strategy='mean')
    data2 = imp_mean.fit_transform(data2)
    scaler = MinMaxScaler((0, 1))
    data2 = scaler.fit_transform(data2)
    features = features + list(f2)
    est = KBinsDiscretizer(n_bins=3, encode='onehot-dense', strategy='uniform')
    data2d = est.fit_transform(data2)
    f2d = []
    for feature in f2:
        # f2d.append(feature + '_VLOW')
        f2d.append(feature + '_LOW')
        f2d.append(feature + '_NORMAL')
        f2d.append(feature + '_HIGH')
        # f2d.append(feature + '_VHIGH')
    features = fs + f2d

    datax = np.hstack((data1, data2d))
    # datay = data['day_28_flg'].values
    # datay = (data['hospital_los_day']>6).values
    # datay = data['hosp_exp_flg'].values

    datay = (data['day_28_flg'].values + data['hosp_exp_flg'].values + data['icu_exp_flg'].values + (1-data['censor_flg'].values)) > 0
    # datay = data['day_28_flg'].values

    # model = DecisionTreeClassifier(max_depth=3)
    # # model = RandomForestClassifier()
    # scores = cross_val_score(model, datax, datay, cv=10)
    # print(scores.mean())

    # datax = np.vstack((datax, datax[datay==1], datax[datay==1]))
    # datay = np.hstack((datay, datay[datay==1], datay[datay==1]))

    x = torch.FloatTensor(datax)
    y = one_hot(torch.tensor(datay).to(torch.long)).to(torch.float)
    return x, y, features
//  [python]
def load_mimic(base_dir: str = './data/'):
    data = pd.read_csv(f'{base_dir}/mimic-ii/full_cohort_data.csv')
    # data.drop('hgb_first')
    fs = [
        'aline_flg',
        'gender_num',
        # 'hosp_exp_flg',
        # 'icu_exp_flg',
        # 'day_28_flg',
        # 'censor_flg',
        'sepsis_flg', 'chf_flg', 'afib_flg',
        'renal_flg', 'liver_flg', 'copd_flg', 'cad_flg', 'stroke_flg',
        'mal_flg', 'resp_flg',
    ]
    features = fs
    data1 = data[fs].values
    imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
    data1 = imp_mean.fit_transform(data1)

    f2 = fs.copy()
    f2.append('day_icu_intime')
    f2.append('service_unit')
    f2.append('day_28_flg')
    f2.append('hospital_los_day')
    f2.append('icu_exp_flg')
    f2.append('hosp_exp_flg')
    f2.append('censor_flg')
    f2.append('mort_day_censored')
    f2 = data.columns.difference(f2)
    data2 = data[f2].values
    imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
    data2 = imp_mean.fit_transform(data2)
    scaler = MinMaxScaler((0, 1))
    data2 = scaler.fit_transform(data2)
    features = features + list(f2)
    est = KBinsDiscretizer(n_bins=3, encode='onehot-dense', strategy='uniform')
    data2d = est.fit_transform(data2)
    f2d = []
    for feature in f2:
        # f2d.append(feature + '_VLOW')
        f2d.append(feature + '_LOW')
        f2d.append(feature + '_NORMAL')
        f2d.append(feature + '_HIGH')
        # f2d.append(feature + '_VHIGH')
    features = fs + f2d

    datax = np.hstack((data1, data2d))
    # datay = data['day_28_flg'].values
    # datay = (data['hospital_los_day']>6).values
    # datay = data['hosp_exp_flg'].values

    datay = (data['day_28_flg'].values + data['hosp_exp_flg'].values + data['icu_exp_flg'].values + (1-data['censor_flg'].values)) > 0
    # datay = data['day_28_flg'].values

    # model = DecisionTreeClassifier(max_depth=3)
    # # model = RandomForestClassifier()
    # scores = cross_val_score(model, datax, datay, cv=10)
    # print(scores.mean())

    # datax = np.vstack((datax, datax[datay==1], datax[datay==1]))
    # datay = np.hstack((datay, datay[datay==1], datay[datay==1]))

    x = torch.FloatTensor(datax)
    y = one_hot(torch.tensor(datay).to(torch.long)).to(torch.float)
    return x, y, features
//  [python]
def get_preprocessor():
    numeric_transformer = Pipeline(
        steps=[
            ("impute", Imputer(strategy="mean", missing_values=np.nan)),
            ("scale", MinMaxScaler()),
        ]
    )

    categorical_transformer = Pipeline(
        steps=[
            ("impute", Imputer(strategy="most_frequent")),
            ("ordinal", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num_pipeline", numeric_transformer, make_column_selector(dtype_include=np.number)),
            ("cat_pipeline", categorical_transformer, make_column_selector(dtype_include=np.object_)),
        ],
        remainder="drop",
        n_jobs=-1,
    )
    return preprocessor
//  [python]
def get_preprocessor():
    numeric_transformer = Pipeline(
        steps=[
            ("impute", SimpleImputer(strategy="mean", missing_values=np.nan)),
            ("scale", MinMaxScaler()),
        ]
    )

    categorical_transformer = Pipeline(
        steps=[
            ("impute", SimpleImputer(strategy="most_frequent")),
            ("ordinal", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num_pipeline", numeric_transformer, make_column_selector(dtype_include=np.number)),
            ("cat_pipeline", categorical_transformer, make_column_selector(dtype_include=np.object_)),
        ],
        remainder="drop",
        n_jobs=-1,
    )
    return preprocessor
//  [python]
def test_permutation_importance_mixed_types_pandas():
    pd = pytest.importorskip("pandas")
    rng = np.random.RandomState(42)
    n_repeats = 5

    # Last column is correlated with y
    X = pd.DataFrame({'col1': [1.0, 2.0, 3.0, np.nan],
                      'col2': ['a', 'b', 'a', 'b']})
    y = np.array([0, 1, 0, 1])

    num_preprocess = make_pipeline(Imputer(), StandardScaler())
    preprocess = ColumnTransformer([
        ('num', num_preprocess, ['col1']),
        ('cat', OneHotEncoder(), ['col2'])
    ])
    clf = make_pipeline(preprocess, LogisticRegression(solver='lbfgs'))
    clf.fit(X, y)

    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
                                    random_state=rng)

    assert result.importances.shape == (X.shape[1], n_repeats)
    # the correlated feature with y is the last column and should
    # have the highest importance
    assert np.all(result.importances_mean[-1] > result.importances_mean[:-1])
//  [python]
def test_permutation_importance_mixed_types_pandas():
    pd = pytest.importorskip("pandas")
    rng = np.random.RandomState(42)
    n_repeats = 5

    # Last column is correlated with y
    X = pd.DataFrame({'col1': [1.0, 2.0, 3.0, np.nan],
                      'col2': ['a', 'b', 'a', 'b']})
    y = np.array([0, 1, 0, 1])

    num_preprocess = make_pipeline(SimpleImputer(), StandardScaler())
    preprocess = ColumnTransformer([
        ('num', num_preprocess, ['col1']),
        ('cat', OneHotEncoder(), ['col2'])
    ])
    clf = make_pipeline(preprocess, LogisticRegression(solver='lbfgs'))
    clf.fit(X, y)

    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
                                    random_state=rng)

    assert result.importances.shape == (X.shape[1], n_repeats)
    # the correlated feature with y is the last column and should
    # have the highest importance
    assert np.all(result.importances_mean[-1] > result.importances_mean[:-1])
//  [python]
def fitted_classifier_pipeline(categorical_classifier_data):
    X_train,  y_train, _, _ = categorical_classifier_data

    numeric_features = ['age', 'fare']
    numeric_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_features = ['embarked', 'sex', 'pclass']
    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='most_frequent')),
        ('ordinal', OrdinalEncoder())])

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', preprocessor),
                        ('classifier', RandomForestClassifier(n_estimators=5, max_depth=3))])

    pipeline.fit(X_train, y_train)
    return pipeline
//  [python]
def fitted_classifier_pipeline(categorical_classifier_data):
    X_train,  y_train, _, _ = categorical_classifier_data

    numeric_features = ['age', 'fare']
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_features = ['embarked', 'sex', 'pclass']
    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('ordinal', OrdinalEncoder())])

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', preprocessor),
                        ('classifier', RandomForestClassifier(n_estimators=5, max_depth=3))])

    pipeline.fit(X_train, y_train)
    return pipeline
//  [python]
def define_pipeline(self):
        logger.info("init pipeline")
        week_of_year_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in WEEKS])
        month_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in MONTHS])
        year_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in YEARS])

        self.week_of_year_categories = [f"week_of_year_{c}" for c in week_of_year_ohe.categories_[0].tolist()]
        self.month_categories = [f"month_{c}" for c in month_ohe.categories_[0].tolist()]
        self.year_categories = [f"year_{c}" for c in year_ohe.categories_[0].tolist()]

        self.week_of_year_ohe_transformer = FunctionTransformer(week_of_year_ohe.transform)
        self.month_ohe_transformer = FunctionTransformer(month_ohe.transform)
        self.year_ohe_transformer = FunctionTransformer(year_ohe.transform)

        base_pipeline = Pipeline(
            [("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None))]
        )
        lag_pipeline = Pipeline(
            [("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None))]
        )
        numerical_pipeline = Pipeline(
            [
                ("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("scaler", MinMaxScaler()),
            ]
        )
        categorical_pipeline = Pipeline(
            [
                ("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", OneHotEncoder(sparse=True, handle_unknown="ignore")),
            ]
        )
        week_of_year_pipeline = Pipeline(
            [
                ("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(week_of_year_ohe.transform)),
            ]
        )
        month_pipeline = Pipeline(
            [
                ("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(month_ohe.transform)),
            ]
        )
        year_pipeline = Pipeline(
            [
                ("imputer", Imputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(year_ohe.transform)),
            ]
        )

        self.pipeline = ColumnTransformer(
            [
                ("bare", base_pipeline, self.bare_columns),
                ("lag", lag_pipeline, self.lag_columns),
                ("numerical", numerical_pipeline, ["item_price"]),
                ("categorical", categorical_pipeline, ["store", "item"]),
                ("week_of_year", week_of_year_pipeline, ["week_of_year"]),
                ("month", month_pipeline, ["month"]),
                ("year", year_pipeline, ["year"]),
            ],
            verbose_feature_names_out=True,
        )
        logger.info(f"pipeline: {self.pipeline}")
//  [python]
def define_pipeline(self):
        logger.info("init pipeline")
        week_of_year_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in WEEKS])
        month_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in MONTHS])
        year_ohe = OneHotEncoder(sparse=True, handle_unknown="ignore").fit([[i] for i in YEARS])

        self.week_of_year_categories = [f"week_of_year_{c}" for c in week_of_year_ohe.categories_[0].tolist()]
        self.month_categories = [f"month_{c}" for c in month_ohe.categories_[0].tolist()]
        self.year_categories = [f"year_{c}" for c in year_ohe.categories_[0].tolist()]

        self.week_of_year_ohe_transformer = FunctionTransformer(week_of_year_ohe.transform)
        self.month_ohe_transformer = FunctionTransformer(month_ohe.transform)
        self.year_ohe_transformer = FunctionTransformer(year_ohe.transform)

        base_pipeline = Pipeline(
            [("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None))]
        )
        lag_pipeline = Pipeline(
            [("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None))]
        )
        numerical_pipeline = Pipeline(
            [
                ("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("scaler", MinMaxScaler()),
            ]
        )
        categorical_pipeline = Pipeline(
            [
                ("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", OneHotEncoder(sparse=True, handle_unknown="ignore")),
            ]
        )
        week_of_year_pipeline = Pipeline(
            [
                ("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(week_of_year_ohe.transform)),
            ]
        )
        month_pipeline = Pipeline(
            [
                ("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(month_ohe.transform)),
            ]
        )
        year_pipeline = Pipeline(
            [
                ("simple_imputer", SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=None)),
                ("one_hot_encoder", FunctionTransformer(year_ohe.transform)),
            ]
        )

        self.pipeline = ColumnTransformer(
            [
                ("bare", base_pipeline, self.bare_columns),
                ("lag", lag_pipeline, self.lag_columns),
                ("numerical", numerical_pipeline, ["item_price"]),
                ("categorical", categorical_pipeline, ["store", "item"]),
                ("week_of_year", week_of_year_pipeline, ["week_of_year"]),
                ("month", month_pipeline, ["month"]),
                ("year", year_pipeline, ["year"]),
            ],
            verbose_feature_names_out=True,
        )
        logger.info(f"pipeline: {self.pipeline}")
//  [python]
def _build_model():
    """Build the model to fit."""
    return Pipeline(steps=[
        ('preprocessor',
         ColumnTransformer(transformers=[('num',
                                          Pipeline(steps=[('imputer',
                                                           Imputer(strategy='median')),
                                                          ('scaler',
                                                           StandardScaler())]),
                                          _NUM_FEATURES),
                                         ('cat', OneHotEncoder(),
                                          _CAT_FEATURES)])),
        ('classifier', RandomForestRegressor(random_state=0, max_depth=7, n_estimators=30))
    ])
//  [python]
def _build_model():
    """Build the model to fit."""
    return Pipeline(steps=[
        ('preprocessor',
         ColumnTransformer(transformers=[('num',
                                          Pipeline(steps=[('imputer',
                                                           SimpleImputer(strategy='median')),
                                                          ('scaler',
                                                           StandardScaler())]),
                                          _NUM_FEATURES),
                                         ('cat', OneHotEncoder(),
                                          _CAT_FEATURES)])),
        ('classifier', RandomForestRegressor(random_state=0, max_depth=7, n_estimators=30))
    ])
//  [python]
def test_imputation_mean_median_error_invalid_type(strategy, dtype):
    X = np.array([["a", "b", 3],
                  [4, "e", 6],
                  ["g", "h", 9]], dtype=dtype)

    with pytest.raises(ValueError, match="non-numeric data"):
        imputer = Imputer(strategy=strategy)
        imputer.fit_transform(X)
//  [python]
def test_imputation_mean_median_error_invalid_type(strategy, dtype):
    X = np.array([["a", "b", 3],
                  [4, "e", 6],
                  ["g", "h", 9]], dtype=dtype)

    with pytest.raises(ValueError, match="non-numeric data"):
        imputer = SimpleImputer(strategy=strategy)
        imputer.fit_transform(X)
//  [python]
def test_imputation_shape():
    # Verify the shapes of the imputed matrix for different strategies.
    X = np.random.randn(10, 2)
    X[::2] = np.nan

    for strategy in ['mean', 'median', 'most_frequent', "constant"]:
        imputer = Imputer(strategy=strategy)
        X_imputed = imputer.fit_transform(sparse.csr_matrix(X))
        assert X_imputed.shape == (10, 2)
        X_imputed = imputer.fit_transform(X)
        assert X_imputed.shape == (10, 2)

        iterative_imputer = IterativeImputer(initial_strategy=strategy)
        X_imputed = iterative_imputer.fit_transform(X)
        assert X_imputed.shape == (10, 2)
//  [python]
def test_imputation_shape():
    # Verify the shapes of the imputed matrix for different strategies.
    X = np.random.randn(10, 2)
    X[::2] = np.nan

    for strategy in ['mean', 'median', 'most_frequent', "constant"]:
        imputer = SimpleImputer(strategy=strategy)
        X_imputed = imputer.fit_transform(sparse.csr_matrix(X))
        assert X_imputed.shape == (10, 2)
        X_imputed = imputer.fit_transform(X)
        assert X_imputed.shape == (10, 2)

        iterative_imputer = IterativeImputer(initial_strategy=strategy)
        X_imputed = iterative_imputer.fit_transform(X)
        assert X_imputed.shape == (10, 2)
//  [python]
def _preprocess_fitting_data(
        self,
        x: np.ndarray | pd.DataFrame,
        y: np.ndarray | pd.Series,
    ) -> Tuple[Tensor, Tensor]:
        if not isinstance(x, (np.ndarray, pd.DataFrame)) and not isinstance(
            y, (np.ndarray, pd.Series)
        ):
            x, y = check_X_y(x, y)
        if not isinstance(x, (np.ndarray, pd.DataFrame)):
            x = check_array(x)
        if not isinstance(y, (np.ndarray, pd.Series)):
            y = np.array(y)
        x = np.array(x).copy()
        y = np.array(y).copy()
        self._cat_features = self.cat_features if self.cat_features is not None else []
        # Impute missing values for numerical features with the mean
        self._num_imputer = Imputer(missing_values=np.nan, strategy="mean")
        if len(x.shape) == 2:
            self._all_feature_idxs = np.arange(x.shape[1])
        else:
            raise ValueError("Reshape your data")
        self._numerical_feature_idxs = np.setdiff1d(
            self._all_feature_idxs, self._cat_features
        )
        if len(self._numerical_feature_idxs) > 0:
            self._num_imputer.fit(x[:, self._numerical_feature_idxs])
            x[:, self._numerical_feature_idxs] = self._num_imputer.transform(
                x[:, self._numerical_feature_idxs]
            )

        if len(self._cat_features) > 0:
            # Impute missing values for categorical features with the most frequent category
            self.cat_imputer = Imputer(
                missing_values=np.nan, strategy="most_frequent"
            )
            self.cat_imputer.fit(x[:, self._cat_features])
            x[:, self._cat_features] = self.cat_imputer.transform(
                x[:, self._cat_features]
            )

            # One-hot encode categorical features
            x = pd.DataFrame(x)
            self.one_hot_encoder = ColumnTransformer(
                transformers=[
                    (
                        "cat",
                        OneHotEncoder(sparse_output=False, handle_unknown="ignore"),
                        self._cat_features,
                    )
                ],
                remainder="passthrough",
            )
            self.one_hot_encoder.fit(x)
            x = self.one_hot_encoder.transform(x)

        x, y = check_X_y(x, y)
        # Standardize data
        self._scaler = StandardScaler()
        self._scaler.fit(x)
        x = self._scaler.transform(x)

        check_classification_targets(y)
        y = column_or_1d(y, warn=True)
        self.n_features_in_ = x.shape[1]
        self.classes_, y = np.unique(y, return_inverse=True)
        return torch.tensor(x, dtype=torch.float).to(self.device), torch.tensor(
            y, dtype=torch.long
        ).to(self.device)
//  [python]
def _preprocess_fitting_data(
        self,
        x: np.ndarray | pd.DataFrame,
        y: np.ndarray | pd.Series,
    ) -> Tuple[Tensor, Tensor]:
        if not isinstance(x, (np.ndarray, pd.DataFrame)) and not isinstance(
            y, (np.ndarray, pd.Series)
        ):
            x, y = check_X_y(x, y)
        if not isinstance(x, (np.ndarray, pd.DataFrame)):
            x = check_array(x)
        if not isinstance(y, (np.ndarray, pd.Series)):
            y = np.array(y)
        x = np.array(x).copy()
        y = np.array(y).copy()
        self._cat_features = self.cat_features if self.cat_features is not None else []
        # Impute missing values for numerical features with the mean
        self._num_imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
        if len(x.shape) == 2:
            self._all_feature_idxs = np.arange(x.shape[1])
        else:
            raise ValueError("Reshape your data")
        self._numerical_feature_idxs = np.setdiff1d(
            self._all_feature_idxs, self._cat_features
        )
        if len(self._numerical_feature_idxs) > 0:
            self._num_imputer.fit(x[:, self._numerical_feature_idxs])
            x[:, self._numerical_feature_idxs] = self._num_imputer.transform(
                x[:, self._numerical_feature_idxs]
            )

        if len(self._cat_features) > 0:
            # Impute missing values for categorical features with the most frequent category
            self.cat_imputer = SimpleImputer(
                missing_values=np.nan, strategy="most_frequent"
            )
            self.cat_imputer.fit(x[:, self._cat_features])
            x[:, self._cat_features] = self.cat_imputer.transform(
                x[:, self._cat_features]
            )

            # One-hot encode categorical features
            x = pd.DataFrame(x)
            self.one_hot_encoder = ColumnTransformer(
                transformers=[
                    (
                        "cat",
                        OneHotEncoder(sparse_output=False, handle_unknown="ignore"),
                        self._cat_features,
                    )
                ],
                remainder="passthrough",
            )
            self.one_hot_encoder.fit(x)
            x = self.one_hot_encoder.transform(x)

        x, y = check_X_y(x, y)
        # Standardize data
        self._scaler = StandardScaler()
        self._scaler.fit(x)
        x = self._scaler.transform(x)

        check_classification_targets(y)
        y = column_or_1d(y, warn=True)
        self.n_features_in_ = x.shape[1]
        self.classes_, y = np.unique(y, return_inverse=True)
        return torch.tensor(x, dtype=torch.float).to(self.device), torch.tensor(
            y, dtype=torch.long
        ).to(self.device)
//  [python]
def test_imputation_const_mostf_error_invalid_types(strategy, dtype):
    # Test imputation on non-numeric data using "most_frequent" and "constant"
    # strategy
    X = np.array([
        [np.nan, np.nan, "a", "f"],
        [np.nan, "c", np.nan, "d"],
        [np.nan, "b", "d", np.nan],
        [np.nan, "c", "d", "h"],
    ], dtype=dtype)

    err_msg = "SimpleImputer does not support data"
    with pytest.raises(ValueError, match=err_msg):
        imputer = Imputer(strategy=strategy)
        imputer.fit(X).transform(X)
//  [python]
def test_imputation_const_mostf_error_invalid_types(strategy, dtype):
    # Test imputation on non-numeric data using "most_frequent" and "constant"
    # strategy
    X = np.array([
        [np.nan, np.nan, "a", "f"],
        [np.nan, "c", np.nan, "d"],
        [np.nan, "b", "d", np.nan],
        [np.nan, "c", "d", "h"],
    ], dtype=dtype)

    err_msg = "SimpleImputer does not support data"
    with pytest.raises(ValueError, match=err_msg):
        imputer = SimpleImputer(strategy=strategy)
        imputer.fit(X).transform(X)
//  [python]
def _check_statistics(X, X_true,
                      strategy, statistics, missing_values):
    """Utility function for testing imputation for a given strategy.

    Test with dense and sparse arrays

    Check that:
        - the statistics (mean, median, mode) are correct
        - the missing values are imputed correctly"""

    err_msg = "Parameters: strategy = %s, missing_values = %s, " \
              "sparse = {0}" % (strategy, missing_values)

    assert_ae = assert_array_equal

    if X.dtype.kind == 'f' or X_true.dtype.kind == 'f':
        assert_ae = assert_array_almost_equal

    # Normal matrix
    imputer = Imputer(missing_values, strategy=strategy)
    X_trans = imputer.fit(X).transform(X.copy())
    assert_ae(imputer.statistics_, statistics,
              err_msg=err_msg.format(False))
    assert_ae(X_trans, X_true, err_msg=err_msg.format(False))

    # Sparse matrix
    imputer = Imputer(missing_values, strategy=strategy)
    imputer.fit(sparse.csc_matrix(X))
    X_trans = imputer.transform(sparse.csc_matrix(X.copy()))

    if sparse.issparse(X_trans):
        X_trans = X_trans.toarray()

    assert_ae(imputer.statistics_, statistics,
              err_msg=err_msg.format(True))
    assert_ae(X_trans, X_true, err_msg=err_msg.format(True))
//  [python]
def _check_statistics(X, X_true,
                      strategy, statistics, missing_values):
    """Utility function for testing imputation for a given strategy.

    Test with dense and sparse arrays

    Check that:
        - the statistics (mean, median, mode) are correct
        - the missing values are imputed correctly"""

    err_msg = "Parameters: strategy = %s, missing_values = %s, " \
              "sparse = {0}" % (strategy, missing_values)

    assert_ae = assert_array_equal

    if X.dtype.kind == 'f' or X_true.dtype.kind == 'f':
        assert_ae = assert_array_almost_equal

    # Normal matrix
    imputer = SimpleImputer(missing_values, strategy=strategy)
    X_trans = imputer.fit(X).transform(X.copy())
    assert_ae(imputer.statistics_, statistics,
              err_msg=err_msg.format(False))
    assert_ae(X_trans, X_true, err_msg=err_msg.format(False))

    # Sparse matrix
    imputer = SimpleImputer(missing_values, strategy=strategy)
    imputer.fit(sparse.csc_matrix(X))
    X_trans = imputer.transform(sparse.csc_matrix(X.copy()))

    if sparse.issparse(X_trans):
        X_trans = X_trans.toarray()

    assert_ae(imputer.statistics_, statistics,
              err_msg=err_msg.format(True))
    assert_ae(X_trans, X_true, err_msg=err_msg.format(True))
//  [python]
def test_imputation_most_frequent_objects(marker):
    # Test imputation using the most-frequent strategy.
    X = np.array([
        [marker, marker, "a", "f"],
        [marker, "c", marker, "d"],
        [marker, "b", "d", marker],
        [marker, "c", "d", "h"],
    ], dtype=object)

    X_true = np.array([
        ["c", "a", "f"],
        ["c", "d", "d"],
        ["b", "d", "d"],
        ["c", "d", "h"],
    ], dtype=object)

    imputer = Imputer(missing_values=marker,
                            strategy="most_frequent")
    X_trans = imputer.fit(X).transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_most_frequent_objects(marker):
    # Test imputation using the most-frequent strategy.
    X = np.array([
        [marker, marker, "a", "f"],
        [marker, "c", marker, "d"],
        [marker, "b", "d", marker],
        [marker, "c", "d", "h"],
    ], dtype=object)

    X_true = np.array([
        ["c", "a", "f"],
        ["c", "d", "d"],
        ["b", "d", "d"],
        ["c", "d", "h"],
    ], dtype=object)

    imputer = SimpleImputer(missing_values=marker,
                            strategy="most_frequent")
    X_trans = imputer.fit(X).transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_integer():
    # Test imputation using the constant strategy on integers
    X = np.array([
        [-1, 2, 3, -1],
        [4, -1, 5, -1],
        [6, 7, -1, -1],
        [8, 9, 0, -1]
    ])

    X_true = np.array([
        [0, 2, 3, 0],
        [4, 0, 5, 0],
        [6, 7, 0, 0],
        [8, 9, 0, 0]
    ])

    imputer = Imputer(missing_values=-1, strategy="constant",
                            fill_value=0)
    X_trans = imputer.fit_transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_integer():
    # Test imputation using the constant strategy on integers
    X = np.array([
        [-1, 2, 3, -1],
        [4, -1, 5, -1],
        [6, 7, -1, -1],
        [8, 9, 0, -1]
    ])

    X_true = np.array([
        [0, 2, 3, 0],
        [4, 0, 5, 0],
        [6, 7, 0, 0],
        [8, 9, 0, 0]
    ])

    imputer = SimpleImputer(missing_values=-1, strategy="constant",
                            fill_value=0)
    X_trans = imputer.fit_transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_most_frequent_pandas(dtype):
    # Test imputation using the most frequent strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["a", "i", "x"],
        ["a", "j", "y"],
        ["a", "j", "x"],
        ["b", "j", "x"]
    ], dtype=object)

    imputer = Imputer(strategy="most_frequent")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_most_frequent_pandas(dtype):
    # Test imputation using the most frequent strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["a", "i", "x"],
        ["a", "j", "y"],
        ["a", "j", "x"],
        ["b", "j", "x"]
    ], dtype=object)

    imputer = SimpleImputer(strategy="most_frequent")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_float(array_constructor):
    # Test imputation using the constant strategy on floats
    X = np.array([
        [np.nan, 1.1, 0, np.nan],
        [1.2, np.nan, 1.3, np.nan],
        [0, 0, np.nan, np.nan],
        [1.4, 1.5, 0, np.nan]
    ])

    X_true = np.array([
        [-1, 1.1, 0, -1],
        [1.2, -1, 1.3, -1],
        [0, 0, -1, -1],
        [1.4, 1.5, 0, -1]
    ])

    X = array_constructor(X)

    X_true = array_constructor(X_true)

    imputer = Imputer(strategy="constant", fill_value=-1)
    X_trans = imputer.fit_transform(X)

    assert_allclose_dense_sparse(X_trans, X_true)
//  [python]
def test_imputation_constant_float(array_constructor):
    # Test imputation using the constant strategy on floats
    X = np.array([
        [np.nan, 1.1, 0, np.nan],
        [1.2, np.nan, 1.3, np.nan],
        [0, 0, np.nan, np.nan],
        [1.4, 1.5, 0, np.nan]
    ])

    X_true = np.array([
        [-1, 1.1, 0, -1],
        [1.2, -1, 1.3, -1],
        [0, 0, -1, -1],
        [1.4, 1.5, 0, -1]
    ])

    X = array_constructor(X)

    X_true = array_constructor(X_true)

    imputer = SimpleImputer(strategy="constant", fill_value=-1)
    X_trans = imputer.fit_transform(X)

    assert_allclose_dense_sparse(X_trans, X_true)