jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
num_pipeline = Pipeline(steps=[
    ('impute', SimpleImputer(strategy='mean')),
    ('scale',MinMaxScaler())])

cat_pipeline = Pipeline(steps=[
        ('impute', SimpleImputer(strategy='most_frequent')),
        ('one-hot',OneHotEncoder(handle_unknown='ignore', sparse=False))
    ])
//  [python]
def test_pipeline_compatibility(estimator: RegressorMixin) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B", "A", "B", "B", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5, 4, 3, np.nan, 3],
            "y": [5, 7, 3, 9, 10, 8, 9, 7, 9, 8]
        }
    )
    y = pd.Series([5, 7, 3, 9, 10, 8, 9, 7, 10, 5])
    X_train_toy, X_calib_toy, y_train_toy, y_calib_toy = train_test_split(
        X,
        y,
        test_size=0.5,
        random_state=random_state
    )
    numeric_preprocessor = Pipeline(
        [
            ("imputer", Imputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(preprocessor, estimator)
    mapie = MapieQuantileRegressor(pipe, alpha=0.4)
    mapie.fit(
        X_train_toy,
        y_train_toy,
        X_calib=X_calib_toy,
        y_calib=y_calib_toy
    )
    mapie.predict(X)
//  [python]
def test_pipeline_compatibility(estimator: RegressorMixin) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B", "A", "B", "B", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5, 4, 3, np.nan, 3],
            "y": [5, 7, 3, 9, 10, 8, 9, 7, 9, 8]
        }
    )
    y = pd.Series([5, 7, 3, 9, 10, 8, 9, 7, 10, 5])
    X_train_toy, X_calib_toy, y_train_toy, y_calib_toy = train_test_split(
        X,
        y,
        test_size=0.5,
        random_state=random_state
    )
    numeric_preprocessor = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(preprocessor, estimator)
    mapie = MapieQuantileRegressor(pipe, alpha=0.4)
    mapie.fit(
        X_train_toy,
        y_train_toy,
        X_calib=X_calib_toy,
        y_calib=y_calib_toy
    )
    mapie.predict(X)
//  [python]
def kmeans(X, k, round_values=True):
    """Summarize a dataset with k mean samples weighted by the number of data points they
    each represent.

    Parameters
    ----------
    X : numpy.array or pandas.DataFrame or any scipy.sparse matrix
        Matrix of data samples to summarize (# samples x # features)

    k : int
        Number of means to use for approximation.

    round_values : bool
        For all i, round the ith dimension of each mean sample to match the nearest value
        from X[:,i]. This ensures discrete features always get a valid value.

    Returns
    -------
    DenseData object.

    """
    group_names = [str(i) for i in range(X.shape[1])]
    if isinstance(X, pd.DataFrame):
        group_names = X.columns
        X = X.values

    # in case there are any missing values in data impute them
    imp = Imputer(missing_values=np.nan, strategy='mean')
    X = imp.fit_transform(X)

    # Specify `n_init` for consistent behaviour between sklearn versions
    kmeans = KMeans(n_clusters=k, random_state=0, n_init=10).fit(X)

    if round_values:
        for i in range(k):
            for j in range(X.shape[1]):
                xj = X[:,j].toarray().flatten() if scipy.sparse.issparse(X) else X[:, j] # sparse support courtesy of @PrimozGodec
                ind = np.argmin(np.abs(xj - kmeans.cluster_centers_[i,j]))
                kmeans.cluster_centers_[i,j] = X[ind,j]
    return DenseData(kmeans.cluster_centers_, group_names, None, 1.0*np.bincount(kmeans.labels_))
//  [python]
def kmeans(X, k, round_values=True):
    """Summarize a dataset with k mean samples weighted by the number of data points they
    each represent.

    Parameters
    ----------
    X : numpy.array or pandas.DataFrame or any scipy.sparse matrix
        Matrix of data samples to summarize (# samples x # features)

    k : int
        Number of means to use for approximation.

    round_values : bool
        For all i, round the ith dimension of each mean sample to match the nearest value
        from X[:,i]. This ensures discrete features always get a valid value.

    Returns
    -------
    DenseData object.

    """
    group_names = [str(i) for i in range(X.shape[1])]
    if isinstance(X, pd.DataFrame):
        group_names = X.columns
        X = X.values

    # in case there are any missing values in data impute them
    imp = SimpleImputer(missing_values=np.nan, strategy='mean')
    X = imp.fit_transform(X)

    # Specify `n_init` for consistent behaviour between sklearn versions
    kmeans = KMeans(n_clusters=k, random_state=0, n_init=10).fit(X)

    if round_values:
        for i in range(k):
            for j in range(X.shape[1]):
                xj = X[:,j].toarray().flatten() if scipy.sparse.issparse(X) else X[:, j] # sparse support courtesy of @PrimozGodec
                ind = np.argmin(np.abs(xj - kmeans.cluster_centers_[i,j]))
                kmeans.cluster_centers_[i,j] = X[ind,j]
    return DenseData(kmeans.cluster_centers_, group_names, None, 1.0*np.bincount(kmeans.labels_))
//  [python]
def load_data(save_directory) -> AbstractDataSource:
        h5_file = os.path.join(save_directory, "achilles.h5")
        if not os.path.exists(h5_file):
            csv_file_path = os.path.join(save_directory, "achilles.csv")
            if not os.path.exists(csv_file_path):
                sp.download_streamed(Achilles.FILE_URL, csv_file_path)
            df = pd.read_csv(csv_file_path)
            gene_names = list(map(lambda x: x.split(" ")[0], df.columns.values.tolist()[1:]))
            data = df.values[:, 1:].astype(float).transpose()

            si = Imputer(missing_values=float("nan"), strategy='mean')
            data = si.fit_transform(data)

            name_converter = HGNCNames(save_directory)
            gene_names = name_converter.update_outdated_gene_names(gene_names)

            data_df = pd.DataFrame(data)
            data_df.index = gene_names
            data_df = data_df.groupby(data_df.index).mean()
            gene_names, data = data_df.index.values.tolist(), data_df.values.astype(np.float32)

            HDF5Tools.save_h5_file(h5_file,
                                   data,
                                   "achilles",
                                   column_names=df["DepMap_ID"].values.tolist(),
                                   row_names=gene_names)
        data_source = HDF5DataSource(h5_file, fill_missing_value=0)
        return data_source
//  [python]
def load_data(save_directory) -> AbstractDataSource:
        h5_file = os.path.join(save_directory, "achilles.h5")
        if not os.path.exists(h5_file):
            csv_file_path = os.path.join(save_directory, "achilles.csv")
            if not os.path.exists(csv_file_path):
                sp.download_streamed(Achilles.FILE_URL, csv_file_path)
            df = pd.read_csv(csv_file_path)
            gene_names = list(map(lambda x: x.split(" ")[0], df.columns.values.tolist()[1:]))
            data = df.values[:, 1:].astype(float).transpose()

            si = SimpleImputer(missing_values=float("nan"), strategy='mean')
            data = si.fit_transform(data)

            name_converter = HGNCNames(save_directory)
            gene_names = name_converter.update_outdated_gene_names(gene_names)

            data_df = pd.DataFrame(data)
            data_df.index = gene_names
            data_df = data_df.groupby(data_df.index).mean()
            gene_names, data = data_df.index.values.tolist(), data_df.values.astype(np.float32)

            HDF5Tools.save_h5_file(h5_file,
                                   data,
                                   "achilles",
                                   column_names=df["DepMap_ID"].values.tolist(),
                                   row_names=gene_names)
        data_source = HDF5DataSource(h5_file, fill_missing_value=0)
        return data_source
//  [python]
def classification_pipeline():
    random_forest = AdaptiveRandomForestClassifier(n_models=10)
    steps = [
        ("Imputer", Imputer(missing_values='NaN', strategy="mean")),
        ("standard_scalar", StandardScaler()),
        ("classifier", random_forest),
    ]
    return SAILPipeline(steps=steps, scoring=[metrics.Accuracy])
//  [python]
def classification_pipeline():
    random_forest = AdaptiveRandomForestClassifier(n_models=10)
    steps = [
        ("Imputer", SimpleImputer(missing_values=np.nan, strategy="mean")),
        ("standard_scalar", StandardScaler()),
        ("classifier", random_forest),
    ]
    return SAILPipeline(steps=steps, scoring=[metrics.Accuracy])
//  [python]
def create_column_transformer():
    num_pipeline = Pipeline(steps=[
        ('impute', Imputer()),
        ('scale', MinMaxScaler())
    ])

    cat_pipeline = Pipeline(steps=[
        ('impute', Imputer()),
        ('one-hot', OneHotEncoder(handle_unknown='ignore', sparse=False))
    ])

    col_trans = ColumnTransformer(transformers=[
        ('num_pipeline', num_pipeline, make_column_selector(dtype_include=np.number)),
        ('cat_pipeline', cat_pipeline, make_column_selector(dtype_include=np.object_))
    ],
        remainder='drop',
        n_jobs=-1)

    return col_trans
//  [python]
def create_column_transformer():
    num_pipeline = Pipeline(steps=[
        ('impute', SimpleImputer()),
        ('scale', MinMaxScaler())
    ])

    cat_pipeline = Pipeline(steps=[
        ('impute', SimpleImputer()),
        ('one-hot', OneHotEncoder(handle_unknown='ignore', sparse=False))
    ])

    col_trans = ColumnTransformer(transformers=[
        ('num_pipeline', num_pipeline, make_column_selector(dtype_include=np.number)),
        ('cat_pipeline', cat_pipeline, make_column_selector(dtype_include=np.object_))
    ],
        remainder='drop',
        n_jobs=-1)

    return col_trans
//  [python]
def fit(self, df_train, y_train=None):
        """Fits NA Encoder.

        Parameters
        ----------
        df_train : pandas dataframe of shape = (n_train, n_features)
            The train dataset with numerical and categorical features.

        y_train : pandas series of shape = (n_train, ), default = None
            The target for classification or regression tasks.

        Returns
        -------
        object
            self

        """
        self.__Lcat = df_train.dtypes[df_train.dtypes == 'object'].index
        self.__Lnum = df_train.dtypes[df_train.dtypes != 'object'].index

        # Dealing with numerical features

        if (self.numerical_strategy in ['mean', 'median', "most_frequent"]):

            self.__imp = Imputer(strategy=self.numerical_strategy)

            if (len(self.__Lnum) != 0):
                self.__imp.fit(df_train[self.__Lnum])
            else:
                pass

        elif ((type(self.numerical_strategy) == int) | (type(self.numerical_strategy) == float)):

            pass

        else:

            raise ValueError("Numerical strategy for NA encoding is not valid")

        # Dealing with categorical features

        if (type(self.categorical_strategy) == str):

            if (self.categorical_strategy == "most_frequent"):

                na_count = df_train[self.__Lcat].isnull().sum()

                for col in na_count[na_count>0].index:

                    try:
                        self.__mode[col] = df_train[col].mode()[0]
                    except:
                        self.__mode[col] = "<NULL>"

            else:
                pass

        else:
            raise ValueError("Categorical strategy for NA encoding is not valid")

        self.__fitOK = True

        return self
//  [python]
def fit(self, df_train, y_train=None):
        """Fits NA Encoder.

        Parameters
        ----------
        df_train : pandas dataframe of shape = (n_train, n_features)
            The train dataset with numerical and categorical features.

        y_train : pandas series of shape = (n_train, ), default = None
            The target for classification or regression tasks.

        Returns
        -------
        object
            self

        """
        self.__Lcat = df_train.dtypes[df_train.dtypes == 'object'].index
        self.__Lnum = df_train.dtypes[df_train.dtypes != 'object'].index

        # Dealing with numerical features

        if (self.numerical_strategy in ['mean', 'median', "most_frequent"]):

            self.__imp = SimpleImputer(strategy=self.numerical_strategy)

            if (len(self.__Lnum) != 0):
                self.__imp.fit(df_train[self.__Lnum])
            else:
                pass

        elif ((type(self.numerical_strategy) == int) | (type(self.numerical_strategy) == float)):

            pass

        else:

            raise ValueError("Numerical strategy for NA encoding is not valid")

        # Dealing with categorical features

        if (type(self.categorical_strategy) == str):

            if (self.categorical_strategy == "most_frequent"):

                na_count = df_train[self.__Lcat].isnull().sum()

                for col in na_count[na_count>0].index:

                    try:
                        self.__mode[col] = df_train[col].mode()[0]
                    except:
                        self.__mode[col] = "<NULL>"

            else:
                pass

        else:
            raise ValueError("Categorical strategy for NA encoding is not valid")

        self.__fitOK = True

        return self
//  [python]
def disparity2depth_psm(disparity):
    baseline = 0.202993
    focal = 2355.722801
    # depth = 250*np.ones(disparity.shape)
    # In SGM there are NAN values, as areas could not be calculated. The SimpleImputer interpolates those wholes.
    # Larger wholes in some cases can not be closed. Check this publication for better results: In Defense of Classical Image Processing: Fast Depth Completion on the CPU
    imp = Imputer(missing_values=np.nan, strategy="mean")
    disparity = imp.fit_transform(disparity) # TODO: this changes the shape of disparity from (412, 960) to (412, 957)
    depth = 250*np.ones(disparity.shape)  # maybe this helps
    depth[disparity == 0] = 250
    depth[disparity != 0] = focal * baseline / disparity[disparity != 0]

    return np.clip(depth,0,250)
//  [python]
def disparity2depth_psm(disparity):
    baseline = 0.202993
    focal = 2355.722801
    # depth = 250*np.ones(disparity.shape)
    # In SGM there are NAN values, as areas could not be calculated. The SimpleImputer interpolates those wholes.
    # Larger wholes in some cases can not be closed. Check this publication for better results: In Defense of Classical Image Processing: Fast Depth Completion on the CPU
    imp = SimpleImputer(missing_values=np.nan, strategy="mean")
    disparity = imp.fit_transform(disparity) # TODO: this changes the shape of disparity from (412, 960) to (412, 957)
    depth = 250*np.ones(disparity.shape)  # maybe this helps
    depth[disparity == 0] = 250
    depth[disparity != 0] = focal * baseline / disparity[disparity != 0]

    return np.clip(depth,0,250)
//  [python]
def regression_pipeline():
    random_forest = AdaptiveRandomForestRegressor()
    steps = [
        ("Imputer", Imputer(missing_values='NaN', strategy="mean")),
        ("standard_scalar", StandardScaler()),
        ("regressor", random_forest),
    ]
    return SAILPipeline(steps=steps, scoring=[metrics.Accuracy])
//  [python]
def regression_pipeline():
    random_forest = AdaptiveRandomForestRegressor()
    steps = [
        ("Imputer", SimpleImputer(missing_values=np.nan, strategy="mean")),
        ("standard_scalar", StandardScaler()),
        ("regressor", random_forest),
    ]
    return SAILPipeline(steps=steps, scoring=[metrics.Accuracy])
//  [python]
def fit(self, X: Union[DataFrame, np.array], y):
        """Fit transformer.

        Args:
            X: A numpy array or a pandas dataframe of training data.
            y: A numpy array or a pandas series of labels.

        Returns:
            X: Processed numpy array or pandas dataframe of training data.
            y: Processed numpy array or pandas series of labels.
        """
        assert isinstance(X, DataFrame)
        X = X.copy()
        n = X.shape[0]

        assert len(self.num_columns) == 0, "Trying to call fit() twice, something is wrong"

        for column in X.columns:
            # sklearn/utils/validation.py needs int/float values
            if X[column].dtype.name in ("object", "category"):
                if (
                    # drop columns where all values are the same
                    X[column].nunique() == 1
                    # this drops UID-type cols
                    or X[column].nunique(dropna=True) == n - X[column].isnull().sum()
                ):
                    self.drop_columns.append(column)
                elif column != self.time_idx:
                    self.cat_columns.append(column)
            elif X[column].nunique(dropna=True) < 2:
                self.drop_columns.append(column)
            elif X[column].dtype.name == "datetime64[ns]":
                pass  # these will be processed at model level,
                # so they can also be done in the predict method
            else:
                self.num_columns.append(column)

        if self.num_columns:
            self.transformer = ColumnTransformer(
                [
                    (
                        "continuous",
                        Imputer(missing_values=np.nan, strategy="median"),
                        self.num_columns,
                    )
                ]
            )

            self.transformer.fit(X[self.num_columns])
        else:
            self.transformer = None

        # TODO: revisit for multivariate series, and recast for a single df input anyway
        if isinstance(y, Series):
            y = y.rename(self.label)

        if isinstance(y, pd.DataFrame):
            ycol = y[y.columns[0]]
        elif isinstance(y, pd.Series):
            ycol = y
        else:
            raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")

        if not pd.api.types.is_numeric_dtype(ycol):
            self.label_transformer = LabelEncoder()
            self.label_transformer.fit(ycol)
        else:
            self.label_transformer = None
//  [python]
def fit(self, X: Union[DataFrame, np.array], y):
        """Fit transformer.

        Args:
            X: A numpy array or a pandas dataframe of training data.
            y: A numpy array or a pandas series of labels.

        Returns:
            X: Processed numpy array or pandas dataframe of training data.
            y: Processed numpy array or pandas series of labels.
        """
        assert isinstance(X, DataFrame)
        X = X.copy()
        n = X.shape[0]

        assert len(self.num_columns) == 0, "Trying to call fit() twice, something is wrong"

        for column in X.columns:
            # sklearn/utils/validation.py needs int/float values
            if X[column].dtype.name in ("object", "category"):
                if (
                    # drop columns where all values are the same
                    X[column].nunique() == 1
                    # this drops UID-type cols
                    or X[column].nunique(dropna=True) == n - X[column].isnull().sum()
                ):
                    self.drop_columns.append(column)
                elif column != self.time_idx:
                    self.cat_columns.append(column)
            elif X[column].nunique(dropna=True) < 2:
                self.drop_columns.append(column)
            elif X[column].dtype.name == "datetime64[ns]":
                pass  # these will be processed at model level,
                # so they can also be done in the predict method
            else:
                self.num_columns.append(column)

        if self.num_columns:
            self.transformer = ColumnTransformer(
                [
                    (
                        "continuous",
                        SimpleImputer(missing_values=np.nan, strategy="median"),
                        self.num_columns,
                    )
                ]
            )

            self.transformer.fit(X[self.num_columns])
        else:
            self.transformer = None

        # TODO: revisit for multivariate series, and recast for a single df input anyway
        if isinstance(y, Series):
            y = y.rename(self.label)

        if isinstance(y, pd.DataFrame):
            ycol = y[y.columns[0]]
        elif isinstance(y, pd.Series):
            ycol = y
        else:
            raise ValueError("y must be either a pd.Series or a pd.DataFrame at this stage")

        if not pd.api.types.is_numeric_dtype(ycol):
            self.label_transformer = LabelEncoder()
            self.label_transformer.fit(ycol)
        else:
            self.label_transformer = None
//  [python]
def init_gmm(features, n_components):
    imp = Imputer(missing_values=np.nan, strategy="mean")
    init_x = imp.fit_transform(features)
    gmm = GaussianMixture(n_components=n_components, covariance_type="diag").fit(init_x)
    return gmm
//  [python]
def init_gmm(features, n_components):
    imp = SimpleImputer(missing_values=np.nan, strategy="mean")
    init_x = imp.fit_transform(features)
    gmm = GaussianMixture(n_components=n_components, covariance_type="diag").fit(init_x)
    return gmm
//  [python]
def preprocessing_pipeline():
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("scaler", MinMaxScaler()),
        ]
    )
    categorical_transformer = OneHotEncoder(
        handle_unknown="ignore", categories=[CATEGORICAL_FEATURES_LIST]
    )
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, NUMERICAL_FEATURES),
            ("cat", categorical_transformer, CATEGORICAL_FEATURES),
        ]
    )
    return preprocessor
//  [python]
def preprocessing_pipeline():
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", MinMaxScaler()),
        ]
    )
    categorical_transformer = OneHotEncoder(
        handle_unknown="ignore", categories=[CATEGORICAL_FEATURES_LIST]
    )
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, NUMERICAL_FEATURES),
            ("cat", categorical_transformer, CATEGORICAL_FEATURES),
        ]
    )
    return preprocessor
//  [python]
def train_preprocessor(data):
    """Train preprocessor."""
    print("Training preprocessor.")

    ordinal_features = [
        n for (n, _) in enumerate(data.feature_names)
        if n not in data.category_map
    ]

    ordinal_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())
    ])

    preprocessor = ColumnTransformer(transformers=[
        ('num', ordinal_transformer, ordinal_features),
    ])

    preprocessor.fit(data.data)

    return  preprocessor
//  [python]
def train_preprocessor(data):
    """Train preprocessor."""
    print("Training preprocessor.")

    ordinal_features = [
        n for (n, _) in enumerate(data.feature_names)
        if n not in data.category_map
    ]

    ordinal_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
    ])

    preprocessor = ColumnTransformer(transformers=[
        ('num', ordinal_transformer, ordinal_features),
    ])

    preprocessor.fit(data.data)

    return  preprocessor
//  [python]
def get_classification_model_pipeline(continuous_features,
                                      categorical_features):
    # We create the preprocessing pipelines for both numeric
    # and categorical data.
    numeric_transformer = Pipeline(steps=[
        ('imputer', Imputer()),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestClassifier())])

    return pipeline
//  [python]
def get_classification_model_pipeline(continuous_features,
                                      categorical_features):
    # We create the preprocessing pipelines for both numeric
    # and categorical data.
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer()),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    transformations = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, continuous_features),
            ('cat', categorical_transformer, categorical_features)])

    # Append classifier to preprocessing pipeline.
    # Now we have a full prediction pipeline.
    pipeline = Pipeline(steps=[('preprocessor', transformations),
                               ('classifier', RandomForestClassifier())])

    return pipeline
//  [python]
def testRegressionMultiTaskFit(self, _, model_name, parameters):
        """Test model training for multitask regression models."""
        parameters["verbose"] = -1
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_regression",
            parameters=parameters,
        )
        self.fitTest(model, dataset)
        predictor = import_module("..pyboost", __name__).PyBoostModel(
            name=f"{model_name}_multitask_regression",
        )
        self.predictorTest(predictor, dataset=dataset)
//  [python]
def testRegressionMultiTaskFit(self, _, model_name, parameters):
        """Test model training for multitask regression models."""
        parameters["verbose"] = -1
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_regression",
            parameters=parameters,
        )
        self.fitTest(model, dataset)
        predictor = import_module("..pyboost", __name__).PyBoostModel(
            name=f"{model_name}_multitask_regression",
        )
        self.predictorTest(predictor, dataset=dataset)
//  [python]
def test_onnx_imputer_const(self, rtol=1e-06, atol=1e-06):
        model = Imputer(strategy="constant")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def test_onnx_imputer_const(self, rtol=1e-06, atol=1e-06):
        model = SimpleImputer(strategy="constant")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def get_column_transformer(numerical: list, categorical: list) -> ColumnTransformer:
    """
    Create a column transformer for the numerical and categorical columns later
    used in the model pipeline.
    :param numerical: Numerical columns in the dataset
    :param categorical: Categorical columns in the dataset
    :return: A new column transformer
    """

    numeric_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='median')),
        ('scale', StandardScaler())
        ])

    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='constant', fill_value='missing')),
        ('onehot', OneHotEncoder(handle_unknown='ignore', drop='if_binary'))])

    return ColumnTransformer(
                transformers=[
                    ('num', numeric_transformer, numerical),
                    ('cat', categorical_transformer, categorical)])
//  [python]
def get_column_transformer(numerical: list, categorical: list) -> ColumnTransformer:
    """
    Create a column transformer for the numerical and categorical columns later
    used in the model pipeline.
    :param numerical: Numerical columns in the dataset
    :param categorical: Categorical columns in the dataset
    :return: A new column transformer
    """

    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median')),
        ('scale', StandardScaler())
        ])

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
        ('onehot', OneHotEncoder(handle_unknown='ignore', drop='if_binary'))])

    return ColumnTransformer(
                transformers=[
                    ('num', numeric_transformer, numerical),
                    ('cat', categorical_transformer, categorical)])
//  [python]
def test_onnx_imputer_const_nan0(self, rtol=1e-06, atol=1e-06):
        model = Imputer(strategy="constant", fill_value=0)
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def test_onnx_imputer_const_nan0(self, rtol=1e-06, atol=1e-06):
        model = SimpleImputer(strategy="constant", fill_value=0)
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def load_adult_census() -> tuple[pd.DataFrame, pd.Series]:
    """Load the adult census dataset from the UCI Machine Learning Repository.

    Original source: https://archive.ics.uci.edu/ml/datasets/adult

    Note:
        The function requires the `sklearn` package to be installed.

    Returns:
        The adult census dataset as a pandas DataFrame.
    """
    from sklearn.compose import ColumnTransformer
    from sklearn.preprocessing import Imputer, OrdinalEncoder, StandardScaler
    from sklearn.pipeline import Pipeline

    dataset = pd.read_csv(GITHUB_DATA_URL + "adult_census.csv")
    class_label = "class"

    num_feature_names = ["age", "capital-gain", "capital-loss", "hours-per-week", "fnlwgt"]
    cat_feature_names = [
        "workclass",
        "education",
        "marital-status",
        "occupation",
        "relationship",
        "race",
        "sex",
        "native-country",
        "education-num",
    ]
    dataset[num_feature_names] = dataset[num_feature_names].apply(pd.to_numeric)
    num_pipeline = Pipeline(
        [("imputer", Imputer(strategy="median")), ("std_scaler", StandardScaler())]
    )
    cat_pipeline = Pipeline(
        [
            ("ordinal_encoder", OrdinalEncoder()),
        ]
    )
    column_transformer = ColumnTransformer(
        [
            ("numerical", num_pipeline, num_feature_names),
            ("categorical", cat_pipeline, cat_feature_names),
        ],
        remainder="passthrough",
    )
    col_names = num_feature_names + cat_feature_names
    col_names += [feature for feature in dataset.columns if feature not in col_names]
    dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
    dataset.dropna(inplace=True)

    x_data = dataset
    y_data = dataset.pop(class_label)

    # transform '>50K' to 1 and '<=50K' to 0
    y_data = y_data.apply(lambda x: 1 if x == ">50K" else 0)

    return x_data, y_data
//  [python]
def load_adult_census() -> tuple[pd.DataFrame, pd.Series]:
    """Load the adult census dataset from the UCI Machine Learning Repository.

    Original source: https://archive.ics.uci.edu/ml/datasets/adult

    Note:
        The function requires the `sklearn` package to be installed.

    Returns:
        The adult census dataset as a pandas DataFrame.
    """
    from sklearn.compose import ColumnTransformer
    from sklearn.impute import SimpleImputer
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OrdinalEncoder, StandardScaler

    dataset = pd.read_csv(GITHUB_DATA_URL + "adult_census.csv")
    class_label = "class"

    num_feature_names = ["age", "capital-gain", "capital-loss", "hours-per-week", "fnlwgt"]
    cat_feature_names = [
        "workclass",
        "education",
        "marital-status",
        "occupation",
        "relationship",
        "race",
        "sex",
        "native-country",
        "education-num",
    ]
    dataset[num_feature_names] = dataset[num_feature_names].apply(pd.to_numeric)
    num_pipeline = Pipeline(
        [("imputer", SimpleImputer(strategy="median")), ("std_scaler", StandardScaler())]
    )
    cat_pipeline = Pipeline(
        [
            ("ordinal_encoder", OrdinalEncoder()),
        ]
    )
    column_transformer = ColumnTransformer(
        [
            ("numerical", num_pipeline, num_feature_names),
            ("categorical", cat_pipeline, cat_feature_names),
        ],
        remainder="passthrough",
    )
    col_names = num_feature_names + cat_feature_names
    col_names += [feature for feature in dataset.columns if feature not in col_names]
    dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
    dataset.dropna(inplace=True)

    x_data = dataset
    y_data = dataset.pop(class_label)

    # transform '>50K' to 1 and '<=50K' to 0
    y_data = y_data.apply(lambda x: 1 if x == ">50K" else 0)

    return x_data, y_data
//  [python]
def preprocess(raw_data_s3_path: str):
    df = pd.read_csv(
        raw_data_s3_path,
        header=None,
        names=feature_columns_names + [label_column],
        dtype=merge_two_dicts(feature_columns_dtype, label_column_dtype),
    )
    numeric_features = list(feature_columns_names)
    numeric_features.remove("sex")
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = ["sex"]
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocess = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    y = df.pop("rings")
    X_pre = preprocess.fit_transform(df)
    y_pre = y.to_numpy().reshape(len(y), 1)

    X = np.concatenate((y_pre, X_pre), axis=1)

    np.random.shuffle(X)
    train, validation, test = np.split(X, [int(0.7 * len(X)), int(0.85 * len(X))])
    return pd.DataFrame(train), pd.DataFrame(validation), pd.DataFrame(test)
//  [python]
def preprocess(raw_data_s3_path: str):
    df = pd.read_csv(
        raw_data_s3_path,
        header=None,
        names=feature_columns_names + [label_column],
        dtype=merge_two_dicts(feature_columns_dtype, label_column_dtype),
    )
    numeric_features = list(feature_columns_names)
    numeric_features.remove("sex")
    numeric_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = ["sex"]
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocess = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    y = df.pop("rings")
    X_pre = preprocess.fit_transform(df)
    y_pre = y.to_numpy().reshape(len(y), 1)

    X = np.concatenate((y_pre, X_pre), axis=1)

    np.random.shuffle(X)
    train, validation, test = np.split(X, [int(0.7 * len(X)), int(0.85 * len(X))])
    return pd.DataFrame(train), pd.DataFrame(validation), pd.DataFrame(test)
//  [python]
def test_onnx_imputer_mean(self, rtol=1e-06, atol=1e-06):
        model = Imputer(strategy="mean", fill_value="nan")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def test_onnx_imputer_mean(self, rtol=1e-06, atol=1e-06):
        model = SimpleImputer(strategy="mean", fill_value="nan")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model)

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
numerical_transformer = Pipeline(
    steps=[
        ("imputer", Imputer(strategy="median")),
        ("normalizer", StandardScaler()),
    ]
)
//  [python]
numerical_columns = ["age", "fare"]
one_hot_columns = ["n_siblings_spouses", "class"]
int_columns = ["sex", "parch", "deck", "embark_town", "alone"]

numerical_transformer = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median")),
        ("normalizer", StandardScaler()),
    ]
)
//  [python]
def testMultiTaskmodel(
        self, _, task: TargetTasks, alg_name: str, random_state: list[int | None]
    ):
        """Test the DNNModel model in one configuration.

        Args:
            task: Task to test.
            alg_name: Name of the algorithm.
        """
        if task == ModelTasks.MULTITASK_REGRESSION:
            target_props = [
                {"name": "fu", "task": TargetTasks.SINGLECLASS, "th": [0.3]},
                {"name": "CL", "task": TargetTasks.SINGLECLASS, "th": [6.5]},
            ]
        else:
            target_props = [
                {
                    "name": "fu",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [0.3],
                    "imputer": Imputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [6.5],
                    "imputer": Imputer(strategy="mean"),
                },
            ]
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            name=f"{alg_name}_{task}",
            target_props=target_props,
            preparation_settings=None,
        )
        dataset.prepareDataset(
            feature_calculators=[SmilesDesc()],
            split=RandomSplit(test_fraction=0.1, dataset=dataset),
        )
        # initialize model for training from class
        alg_name = f"{alg_name}_{task}"
        model = self.getModel(
            name=alg_name,
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = ChempropModel(name=alg_name, base_dir=model.baseDir)
        self.predictorTest(predictor, dataset=dataset)

        # make predictions with the trained model and check if the results are (not)
        # equal if the random state is the (not) same and check if the output is the
        # same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=alg_name,
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor, # model loaded from file
                dataset=dataset,
                comparison_model=comparison_model, # model not loaded from file
                expect_equal_result=random_state[0] == random_state[1],
            )
        else:
            self.predictorTest(predictor, dataset=dataset)
//  [python]
def testMultiTaskmodel(
        self, _, task: TargetTasks, alg_name: str, random_state: list[int | None]
    ):
        """Test the DNNModel model in one configuration.

        Args:
            task: Task to test.
            alg_name: Name of the algorithm.
        """
        if task == ModelTasks.MULTITASK_REGRESSION:
            target_props = [
                {"name": "fu", "task": TargetTasks.SINGLECLASS, "th": [0.3]},
                {"name": "CL", "task": TargetTasks.SINGLECLASS, "th": [6.5]},
            ]
        else:
            target_props = [
                {
                    "name": "fu",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [0.3],
                    "imputer": SimpleImputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [6.5],
                    "imputer": SimpleImputer(strategy="mean"),
                },
            ]
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            name=f"{alg_name}_{task}",
            target_props=target_props,
            preparation_settings=None,
        )
        dataset.prepareDataset(
            feature_calculators=[SmilesDesc()],
            split=RandomSplit(test_fraction=0.1, dataset=dataset),
        )
        # initialize model for training from class
        alg_name = f"{alg_name}_{task}"
        model = self.getModel(
            name=alg_name,
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = ChempropModel(name=alg_name, base_dir=model.baseDir)
        self.predictorTest(predictor, dataset=dataset)

        # make predictions with the trained model and check if the results are (not)
        # equal if the random state is the (not) same and check if the output is the
        # same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=alg_name,
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor, # model loaded from file
                dataset=dataset,
                comparison_model=comparison_model, # model not loaded from file
                expect_equal_result=random_state[0] == random_state[1],
            )
        else:
            self.predictorTest(predictor, dataset=dataset)
//  [python]
def test_onnx_imputer_converter_raises_rt(self):
        warnings.filterwarnings("ignore")
        model = Imputer(strategy="mean", fill_value="nan")
        X = np.array([[1, 2], [np.nan, 3], [7, 6]], dtype=np.float32)
        model.fit(X)

        # Create ONNX-ML model
        onnx_ml_model = convert_sklearn(model, initial_types=[("float_input", FloatTensorType_onnx(X.shape))])
        onnx_ml_model.graph.node[0].attribute[0].name = "".encode()

        self.assertRaises(RuntimeError, convert, onnx_ml_model, "onnx", X)
//  [python]
def test_onnx_imputer_converter_raises_rt(self):
        warnings.filterwarnings("ignore")
        model = SimpleImputer(strategy="mean", fill_value="nan")
        X = np.array([[1, 2], [np.nan, 3], [7, 6]], dtype=np.float32)
        model.fit(X)

        # Create ONNX-ML model
        onnx_ml_model = convert_sklearn(model, initial_types=[("float_input", FloatTensorType_onnx(X.shape))])
        onnx_ml_model.graph.node[0].attribute[0].name = "".encode()

        self.assertRaises(RuntimeError, convert, onnx_ml_model, "onnx", X)
//  [python]
def test_onnx_imputer_torch(self, rtol=1e-06, atol=1e-06):
        model = Imputer(strategy="constant")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model, mode="torch")

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def test_onnx_imputer_torch(self, rtol=1e-06, atol=1e-06):
        model = SimpleImputer(strategy="constant")
        onnx_ml_pred, onnx_pred = self._test_imputer_converter(model, mode="torch")

        # Check that predicted values match
        np.testing.assert_allclose(onnx_ml_pred, onnx_pred, rtol=rtol, atol=atol)
//  [python]
def imputer(df: pd.DataFrame,
            columns_to_impute: List[str],
            impute_strategy: str = 'median',
            placeholder_value: Optional[Any] = None) -> LearnerReturnType:
    """
    Fits a missing value imputer to the dataset.

    Parameters
    ----------

    df : pandas.DataFrame
        A Pandas' DataFrame with columns to impute missing values.
        It must contain all columns listed in `columns_to_impute`

    columns_to_impute : List of strings
        A list of names of the columns for missing value imputation.

    impute_strategy : String, (default="median")
        The imputation strategy.
        - If "mean", then replace missing values using the mean along the axis.
        - If "median", then replace missing values using the median along the axis.
        - If "most_frequent", then replace missing using the most frequent value along the axis.

    placeholder_value : Any, (default=None)
        if not None, use this as default value when some features only contains
        NA values on training. For transformation, NA values on those features
        will be replaced by `fill_value`.
    """

    if placeholder_value is not None:
        mask_feat_is_na = df[columns_to_impute].isna().all(axis=0)
        columns_to_fill = mask_feat_is_na[mask_feat_is_na].index.values
        columns_imputable = mask_feat_is_na[~mask_feat_is_na].index.values

        fill_fn, _, fill_logs = placeholder_imputer(
            df, columns_to_impute=columns_to_fill, placeholder_value=placeholder_value)
    else:
        columns_to_fill = list()
        columns_imputable = columns_to_impute
        fill_fn, _, fill_logs = identity, None, dict()

    imp = Imputer(strategy=impute_strategy)

    imp.fit(df[columns_imputable].values)

    def p(new_data_set: pd.DataFrame) -> pd.DataFrame:
        new_data = imp.transform(new_data_set[columns_imputable])
        new_cols = pd.DataFrame(data=new_data, columns=columns_imputable).to_dict('list')
        return fill_fn(new_data_set.assign(**new_cols))

    p.__doc__ = learner_pred_fn_docstring("imputer")

    log = {
        'imputer': {
            'impute_strategy': impute_strategy,
            'placeholder_value': placeholder_value,
            'columns_to_impute': columns_to_impute,
            'columns_to_fill': columns_to_fill,
            'columns_imputable': columns_imputable,
            'training_proportion_of_nulls': df[columns_to_impute].isnull().mean(axis=0).to_dict(),
            'statistics': imp.statistics_,
            'placeholder_imputer_fn': fill_fn,
            'placeholder_imputer_logs': fill_logs,
        }
    }

    return p, p(df), log