jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def imputer(df: pd.DataFrame,
            columns_to_impute: List[str],
            impute_strategy: str = 'median',
            placeholder_value: Optional[Any] = None) -> LearnerReturnType:
    """
    Fits a missing value imputer to the dataset.

    Parameters
    ----------

    df : pandas.DataFrame
        A Pandas' DataFrame with columns to impute missing values.
        It must contain all columns listed in `columns_to_impute`

    columns_to_impute : List of strings
        A list of names of the columns for missing value imputation.

    impute_strategy : String, (default="median")
        The imputation strategy.
        - If "mean", then replace missing values using the mean along the axis.
        - If "median", then replace missing values using the median along the axis.
        - If "most_frequent", then replace missing using the most frequent value along the axis.

    placeholder_value : Any, (default=None)
        if not None, use this as default value when some features only contains
        NA values on training. For transformation, NA values on those features
        will be replaced by `fill_value`.
    """

    if placeholder_value is not None:
        mask_feat_is_na = df[columns_to_impute].isna().all(axis=0)
        columns_to_fill = mask_feat_is_na[mask_feat_is_na].index.values
        columns_imputable = mask_feat_is_na[~mask_feat_is_na].index.values

        fill_fn, _, fill_logs = placeholder_imputer(
            df, columns_to_impute=columns_to_fill, placeholder_value=placeholder_value)
    else:
        columns_to_fill = list()
        columns_imputable = columns_to_impute
        fill_fn, _, fill_logs = identity, None, dict()

    imp = SimpleImputer(strategy=impute_strategy)

    imp.fit(df[columns_imputable].values)

    def p(new_data_set: pd.DataFrame) -> pd.DataFrame:
        new_data = imp.transform(new_data_set[columns_imputable])
        new_cols = pd.DataFrame(data=new_data, columns=columns_imputable).to_dict('list')
        return fill_fn(new_data_set.assign(**new_cols))

    p.__doc__ = learner_pred_fn_docstring("imputer")

    log = {
        'imputer': {
            'impute_strategy': impute_strategy,
            'placeholder_value': placeholder_value,
            'columns_to_impute': columns_to_impute,
            'columns_to_fill': columns_to_fill,
            'columns_imputable': columns_imputable,
            'training_proportion_of_nulls': df[columns_to_impute].isnull().mean(axis=0).to_dict(),
            'statistics': imp.statistics_,
            'placeholder_imputer_fn': fill_fn,
            'placeholder_imputer_logs': fill_logs,
        }
    }

    return p, p(df), log
//  [python]
def make_classifer_pipeline(params: Dict[str, Union[str, int]]) -> Pipeline:
    """Create sklearn pipeline to apply transforms and a final estimator"""
    # categorical features transformations
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="most_frequent")),
            (
                "ohe",
                OneHotEncoder(
                    handle_unknown="ignore",
                ),
            ),
        ]
    )

    # numeric features transformations
    numeric_transformer = Pipeline(
        steps=[("imputer", Imputer(strategy="median"))]
    )

    # preprocessing pipeline
    preprocessor = ColumnTransformer(
        transformers=[
            ("numeric", numeric_transformer, NUMERIC_FEATURES),
            ("categorical", categorical_transformer, CATEGORICAL_FEATURES),
        ]
    )

    # model training pipeline
    classifer_pipeline = Pipeline(
        [
            ("preprocessor", preprocessor),
            ("classifier", RandomForestClassifier(**params, n_jobs=-1)),
        ]
    )

    return classifer_pipeline
//  [python]
def make_classifer_pipeline(params: Dict[str, Union[str, int]]) -> Pipeline:
    """Create sklearn pipeline to apply transforms and a final estimator"""
    # categorical features transformations
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="most_frequent")),
            (
                "ohe",
                OneHotEncoder(
                    handle_unknown="ignore",
                ),
            ),
        ]
    )

    # numeric features transformations
    numeric_transformer = Pipeline(
        steps=[("imputer", SimpleImputer(strategy="median"))]
    )

    # preprocessing pipeline
    preprocessor = ColumnTransformer(
        transformers=[
            ("numeric", numeric_transformer, NUMERIC_FEATURES),
            ("categorical", categorical_transformer, CATEGORICAL_FEATURES),
        ]
    )

    # model training pipeline
    classifer_pipeline = Pipeline(
        [
            ("preprocessor", preprocessor),
            ("classifier", RandomForestClassifier(**params, n_jobs=-1)),
        ]
    )

    return classifer_pipeline
//  [python]
def testImputation(self):
        """Test the imputation of missing values in the target properties."""
        self.dataset = QSPRDataset(
            "TestImputation",
            target_props=[
                {
                    "name": "y",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
                {
                    "name": "z",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
            ],
            df=self.df,
            store_dir=self.generatedPath,
            n_jobs=self.nCPU,
            chunk_size=self.chunkSize,
        )
        self.assertEqual(self.dataset.targetProperties[0].name, "y")
        self.assertEqual(self.dataset.targetProperties[1].name, "z")
        self.assertTrue("y_before_impute" in self.dataset.df.columns)
        self.assertTrue("z_before_impute" in self.dataset.df.columns)
        self.assertEqual(self.dataset.df["y"].isna().sum(), 0)
        self.assertEqual(self.dataset.df["z"].isna().sum(), 0)
//  [python]
def testImputation(self):
        """Test the imputation of missing values in the target properties."""
        self.dataset = QSPRDataset(
            "TestImputation",
            target_props=[
                {
                    "name": "y",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
                {
                    "name": "z",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
            ],
            df=self.df,
            store_dir=self.generatedPath,
            n_jobs=self.nCPU,
            chunk_size=self.chunkSize,
        )
        self.assertEqual(self.dataset.targetProperties[0].name, "y")
        self.assertEqual(self.dataset.targetProperties[1].name, "z")
        self.assertTrue("y_before_impute" in self.dataset.df.columns)
        self.assertTrue("z_before_impute" in self.dataset.df.columns)
        self.assertEqual(self.dataset.df["y"].isna().sum(), 0)
        self.assertEqual(self.dataset.df["z"].isna().sum(), 0)
//  [python]
def test_missing_check():
    df = load_chicken(as_frame=True)

    X, y = df.drop(columns="weight"), df["weight"]
    # create missing value
    X.loc[0, "chick"] = np.nan
    model = make_pipeline(Imputer(), LinearRegression())

    # Should not raise error, check is disabled
    m = GroupedPredictor(model, groups=["diet"], check_X=False).fit(X, y)
    m.predict(X)

    # Should raise error, check is still enabled
    with pytest.raises(ValueError) as e:
        GroupedPredictor(model, groups=["diet"]).fit(X, y)
        assert "contains NaN" in str(e)
//  [python]
def test_missing_check():
    df = load_chicken(as_frame=True)

    X, y = df.drop(columns="weight"), df["weight"]
    # create missing value
    X.loc[0, "chick"] = np.nan
    model = make_pipeline(SimpleImputer(), LinearRegression())

    # Should not raise error, check is disabled
    m = GroupedPredictor(model, groups=["diet"], check_X=False).fit(X, y)
    m.predict(X)

    # Should raise error, check is still enabled
    with pytest.raises(ValueError) as e:
        GroupedPredictor(model, groups=["diet"]).fit(X, y)
        assert "contains NaN" in str(e)
//  [python]
def PCAFractionOfComponentsFor95PercentVariance(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        # replace missing values using the mean along each column
        imp_mean = Imputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                sum_ = 0.
                idx = 0
                while sum_ < 0.95 and idx < len(pca.explained_variance_ratio_):
                    sum_ += pca.explained_variance_ratio_[idx]
                    idx += 1
                return float(idx)/float(X.shape[1])

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
def PCAFractionOfComponentsFor95PercentVariance(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        # replace missing values using the mean along each column
        imp_mean = SimpleImputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                sum_ = 0.
                idx = 0
                while sum_ < 0.95 and idx < len(pca.explained_variance_ratio_):
                    sum_ += pca.explained_variance_ratio_[idx]
                    idx += 1
                return float(idx)/float(X.shape[1])

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
def PCAKurtosisFirstPC(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        imp_mean = Imputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                components = pca.components_
                pca.components_ = components[:1]
                transformed = pca.transform(X_transformed)
                pca.components_ = components
                kurtosis = scStat.kurtosis(transformed)
                return kurtosis[0]

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
def PCAKurtosisFirstPC(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        imp_mean = SimpleImputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                components = pca.components_
                pca.components_ = components[:1]
                transformed = pca.transform(X_transformed)
                pca.components_ = components
                kurtosis = scStat.kurtosis(transformed)
                return kurtosis[0]

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
def build_model(hp):
    numerical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("normalizer", StandardScaler()),
        ]
    )

    one_hot_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", fill_value="None")),
            ("one_hot_encoder", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    int_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", fill_value="None")),
            (
                "label_encoder",
                OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1),
            ),
        ]
    )

    transformers = [
        ("numerical", numerical_transformer, numerical_columns),
        ("one_hot", one_hot_transformer, one_hot_columns),
        ("int", int_transformer, int_columns),
    ]
    for index, (col1, col2) in enumerate(cat_column_pairs):
        if not hp.Boolean("combine_{i}".format(i=index)):
            continue
        col1 = str(col1)
        col2 = str(col2)
        transformers.append(
            (col1 + col2, CategoricalCombination(col1, col2), [col1, col2])
        )

    for index, (col1, col2) in enumerate(mean_column_pairs):
        if not hp.Boolean("mean_{i}".format(i=index)):
            continue
        col1 = str(col1)
        col2 = str(col2)
        transformers.append((col1 + col2, MeanEncoder(col1, col2), [col1, col2]))
    print(transformers)
    pipeline = Pipeline(
        steps=[
            ("preprocessing", ColumnTransformer(transformers=transformers)),
            ("impute", Imputer(strategy="median")),
            (
                "model_selection",
                SelectKBest(
                    mutual_info_classif,
                    k=hp.Int("best_k", 5, 13 + len(transformers) - 3),
                ),
            ),
            ("model", SVC()),
        ]
    )

    return pipeline
//  [python]
def build_model(hp):
    numerical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("normalizer", StandardScaler()),
        ]
    )

    one_hot_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", fill_value="None")),
            ("one_hot_encoder", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    int_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", fill_value="None")),
            (
                "label_encoder",
                OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1),
            ),
        ]
    )

    transformers = [
        ("numerical", numerical_transformer, numerical_columns),
        ("one_hot", one_hot_transformer, one_hot_columns),
        ("int", int_transformer, int_columns),
    ]
    for index, (col1, col2) in enumerate(cat_column_pairs):
        if not hp.Boolean("combine_{i}".format(i=index)):
            continue
        col1 = str(col1)
        col2 = str(col2)
        transformers.append(
            (col1 + col2, CategoricalCombination(col1, col2), [col1, col2])
        )

    for index, (col1, col2) in enumerate(mean_column_pairs):
        if not hp.Boolean("mean_{i}".format(i=index)):
            continue
        col1 = str(col1)
        col2 = str(col2)
        transformers.append((col1 + col2, MeanEncoder(col1, col2), [col1, col2]))
    print(transformers)
    pipeline = Pipeline(
        steps=[
            ("preprocessing", ColumnTransformer(transformers=transformers)),
            ("impute", SimpleImputer(strategy="median")),
            (
                "model_selection",
                SelectKBest(
                    mutual_info_classif,
                    k=hp.Int("best_k", 5, 13 + len(transformers) - 3),
                ),
            ),
            ("model", SVC()),
        ]
    )

    return pipeline
//  [python]
def test_missing_values_is_resilience(
    make_data, Tree, sample_weight_train, global_random_seed
):
    """Check that trees can deal with missing values have decent performance."""
    n_samples, n_features = 5_000, 10
    X, y = make_data(
        n_samples=n_samples, n_features=n_features, random_state=global_random_seed
    )

    X_missing = X.copy()
    rng = np.random.RandomState(global_random_seed)
    X_missing[rng.choice([False, True], size=X.shape, p=[0.9, 0.1])] = np.nan
    X_missing_train, X_missing_test, y_train, y_test = train_test_split(
        X_missing, y, random_state=global_random_seed
    )
    if sample_weight_train == "ones":
        sample_weight = np.ones(X_missing_train.shape[0])
    else:
        sample_weight = None

    native_tree = Tree(max_depth=10, random_state=global_random_seed)
    native_tree.fit(X_missing_train, y_train, sample_weight=sample_weight)
    score_native_tree = native_tree.score(X_missing_test, y_test)

    tree_with_imputer = make_pipeline(
        Imputer(), Tree(max_depth=10, random_state=global_random_seed)
    )
    tree_with_imputer.fit(X_missing_train, y_train)
    score_tree_with_imputer = tree_with_imputer.score(X_missing_test, y_test)

    assert (
        score_native_tree > score_tree_with_imputer
    ), f"{score_native_tree=} should be strictly greater than {score_tree_with_imputer}"
//  [python]
def test_missing_values_is_resilience(
    make_data, Tree, sample_weight_train, global_random_seed
):
    """Check that trees can deal with missing values have decent performance."""
    n_samples, n_features = 5_000, 10
    X, y = make_data(
        n_samples=n_samples, n_features=n_features, random_state=global_random_seed
    )

    X_missing = X.copy()
    rng = np.random.RandomState(global_random_seed)
    X_missing[rng.choice([False, True], size=X.shape, p=[0.9, 0.1])] = np.nan
    X_missing_train, X_missing_test, y_train, y_test = train_test_split(
        X_missing, y, random_state=global_random_seed
    )
    if sample_weight_train == "ones":
        sample_weight = np.ones(X_missing_train.shape[0])
    else:
        sample_weight = None

    native_tree = Tree(max_depth=10, random_state=global_random_seed)
    native_tree.fit(X_missing_train, y_train, sample_weight=sample_weight)
    score_native_tree = native_tree.score(X_missing_test, y_test)

    tree_with_imputer = make_pipeline(
        SimpleImputer(), Tree(max_depth=10, random_state=global_random_seed)
    )
    tree_with_imputer.fit(X_missing_train, y_train)
    score_tree_with_imputer = tree_with_imputer.score(X_missing_test, y_test)

    assert (
        score_native_tree > score_tree_with_imputer
    ), f"{score_native_tree=} should be strictly greater than {score_tree_with_imputer}"
//  [python]
def PCASkewnessFirstPC(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        imp_mean = Imputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                components = pca.components_
                pca.components_ = components[:1]
                transformed = pca.transform(X_transformed)
                pca.components_ = components
                skewness = scStat.skew(transformed)
                return skewness[0]

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
def PCASkewnessFirstPC(X):
        pca = sklearn.decomposition.PCA(copy=True)
        rs = np.random.RandomState(42)
        indices = np.arange(X.shape[0])

        imp_mean = SimpleImputer(strategy='mean')
        X_transformed = imp_mean.fit_transform(X)

        for i in range(10):
            try:
                rs.shuffle(indices)
                pca.fit(X_transformed[indices])
                components = pca.components_
                pca.components_ = components[:1]
                transformed = pca.transform(X_transformed)
                pca.components_ = components
                skewness = scStat.skew(transformed)
                return skewness[0]

            except LinAlgError as e:
                pass

        print("Failed to compute PCA")
        #self.logger.warning("Failed to compute a Principle Component Analysis")
        return np.nan
//  [python]
data = pd.read_csv("./input/data.csv")

# Identify columns with missing values
cols_with_missing = [col for col in data.columns if data[col].isnull().any()]

# Initialize the SimpleImputer with mean strategy
imputer = Imputer(strategy="mean")
//  [python]
data = pd.read_csv("./input/data.csv")

# Identify columns with missing values
cols_with_missing = [col for col in data.columns if data[col].isnull().any()]

# Initialize the SimpleImputer with mean strategy
imputer = SimpleImputer(strategy="mean")
//  [python]
def handle_missing_values(df, fill_value=np.nan, strategy="mean"):
    logger.info(
        f"Check for missing values in the dataset ...  \n"
        f"{df.isna().sum()}  \n "
        f"{'-'*100}"
    )

    if strategy.lower() == "drop":
        return df.dropna()

    cleaner = Imputer(fill_value=fill_value, strategy=strategy)
    cleaned = cleaner.fit_transform(df)

    return pd.DataFrame(cleaned, columns=df.columns)
//  [python]
def handle_missing_values(df, fill_value=np.nan, strategy="mean"):
    logger.info(
        f"Check for missing values in the dataset ...  \n"
        f"{df.isna().sum()}  \n "
        f"{'-'*100}"
    )

    if strategy.lower() == "drop":
        return df.dropna()

    cleaner = SimpleImputer(fill_value=fill_value, strategy=strategy)
    cleaned = cleaner.fit_transform(df)

    return pd.DataFrame(cleaned, columns=df.columns)
//  [python]
def __init__(self):

        # Imputation
        self.imp = Imputer(strategy='median')

        # Capping and flooring
        self.cp = _CapFloor(p_low=0.005, p_high=0.995)

        # Scaler
        self.scaler = StandardScaler()

        # Initialize fit status
        self.is_fit = False
//  [python]
def __init__(self):

        # Imputation
        self.imp = SimpleImputer(strategy='median')

        # Capping and flooring
        self.cp = _CapFloor(p_low=0.005, p_high=0.995)

        # Scaler
        self.scaler = StandardScaler()

        # Initialize fit status
        self.is_fit = False
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, "survived"] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns="survived")
        self.y = data.survived.values

        numeric_features = ["age", "fare", "sibsp", "parch"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["gender", "class", "embarked"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                (
                    "classifier",
                    MLPClassifier(
                        hidden_layer_sizes=(50, 100, 50), max_iter=400, random_state=0
                    ),
                ),
            ]
        )

        clf.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, verbose=False)
        self.exp2 = dx.Explainer(clf, self.X, self.y, label="model2", verbose=False)
        self.aspect = dx.Aspect(self.exp)
        self.aspect2 = dx.Aspect(self.exp2, depend_method="pps")
//  [python]
def setUp(self):
        data = dx.datasets.load_titanic()
        data.loc[:, "survived"] = LabelEncoder().fit_transform(data.survived)

        self.X = data.drop(columns="survived")
        self.y = data.survived.values

        numeric_features = ["age", "fare", "sibsp", "parch"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["gender", "class", "embarked"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                (
                    "classifier",
                    MLPClassifier(
                        hidden_layer_sizes=(50, 100, 50), max_iter=400, random_state=0
                    ),
                ),
            ]
        )

        clf.fit(self.X, self.y)

        self.exp = dx.Explainer(clf, self.X, self.y, verbose=False)
        self.exp2 = dx.Explainer(clf, self.X, self.y, label="model2", verbose=False)
        self.aspect = dx.Aspect(self.exp)
        self.aspect2 = dx.Aspect(self.exp2, depend_method="pps")
//  [python]
def build_column(self, data):
        imputer_type = self.cfg["type"]
        if imputer_type == "iterative":
            try:
                from sklearn.experimental import enable_iterative_imputer  # noqa
                from sklearn.impute import IterativeImputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.21.0 installed in order to use the Iterative Imputer!"
                )
            imputer = IterativeImputer()
        elif imputer_type == "knn":
            try:
                from sklearn.impute import KNNImputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.22.0 installed in order to use the Iterative Imputer!"
                )
            n_neighbors = self.cfg.get("n_neighbors") or 2
            imputer = KNNImputer(n_neighbors=n_neighbors)
        elif imputer_type == "simple":
            try:
                from sklearn.preprocessing import Imputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.20.0 installed in order to use the Iterative Imputer!"
                )
            imputer = Imputer()
        else:
            raise NotImplementedError(
                "'{}' sklearn imputer not implemented yet!".format(imputer_type)
            )
        output = imputer.fit_transform(data[[self.col]])
        return pd.DataFrame(output, columns=[self.col], index=data.index)[self.col]
//  [python]
def build_column(self, data):
        imputer_type = self.cfg["type"]
        if imputer_type == "iterative":
            try:
                from sklearn.experimental import enable_iterative_imputer  # noqa
                from sklearn.impute import IterativeImputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.21.0 installed in order to use the Iterative Imputer!"
                )
            imputer = IterativeImputer()
        elif imputer_type == "knn":
            try:
                from sklearn.impute import KNNImputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.22.0 installed in order to use the Iterative Imputer!"
                )
            n_neighbors = self.cfg.get("n_neighbors") or 2
            imputer = KNNImputer(n_neighbors=n_neighbors)
        elif imputer_type == "simple":
            try:
                from sklearn.impute import SimpleImputer
            except ImportError:
                raise Exception(
                    "You must have at least scikit-learn 0.20.0 installed in order to use the Iterative Imputer!"
                )
            imputer = SimpleImputer()
        else:
            raise NotImplementedError(
                "'{}' sklearn imputer not implemented yet!".format(imputer_type)
            )
        output = imputer.fit_transform(data[[self.col]])
        return pd.DataFrame(output, columns=[self.col], index=data.index)[self.col]
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_nfl()

        model = Pipeline([
            ('imputer', Imputer(missing_values='NaN', strategy='mean')),
            ('icdm', InterclusterDistance(KMeans(5, random_state=42), random_state=42))
        ])

        model.fit(X)
        model['icdm'].finalize()
        self.assert_images_similar(model['icdm'], tol=2.0)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_nfl()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('icdm', InterclusterDistance(KMeans(5, random_state=42), random_state=42))
        ])

        model.fit(X)
        model['icdm'].finalize()
        self.assert_images_similar(model['icdm'], tol=2.0)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_nfl()

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('icdm', intercluster_distance(KMeans(5, random_state=42), X, random_state=42))
        ])

        model['icdm'].finalize()
        self.assert_images_similar(model['icdm'], tol=2.0)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_nfl()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('icdm', intercluster_distance(KMeans(5, random_state=42), X, random_state=42))
        ])

        model['icdm'].finalize()
        self.assert_images_similar(model['icdm'], tol=2.0)
//  [python]
def impute_values(feature, dt, sentinel):
    r"""Impute values for a given data type. The *median* strategy
    is applied for floating point values, and the *most frequent*
    strategy is applied for integer or Boolean values.

    Parameters
    ----------
    feature : pandas.Series or numpy.array
        The feature for imputation.
    dt : str
        The values ``'float64'``, ``'int64'``, or ``'bool'``.
    sentinel : float
        The number to be imputed for NaN values.

    Returns
    -------
    imputed : numpy.array
        The feature after imputation.

    Raises
    ------
    TypeError
        Data type ``dt`` is invalid for imputation.

    References
    ----------
    You can find more information on feature imputation here [IMP]_.

    .. [IMP] http://scikit-learn.org/stable/modules/preprocessing.html#imputation

    """

    try:
        # for pandas series
        feature = feature.values.reshape(-1, 1)
    except:
        # for numpy array
        feature = feature.reshape(-1, 1)

    if dt == 'float64':
        logger.info("    Imputation for Data Type %s: Median Strategy" % dt)
        # replace infinity with imputed value
        feature[np.isinf(feature)] = np.nan
        imp = Imputer(missing_values=np.nan, strategy='median')
    elif dt == 'int64':
        logger.info("    Imputation for Data Type %s: Most Frequent Strategy" % dt)
        imp = Imputer(missing_values=np.nan, strategy='most_frequent')
    elif dt != 'bool':
        logger.info("    Imputation for Data Type %s: Fill Strategy with %d" % (dt, sentinel))
        imp = Imputer(missing_values=np.nan, strategy='constant', fill_value=sentinel)
    else:
        logger.info("    No Imputation for Data Type %s" % dt)
        imp = None

    imputed = imp.fit_transform(feature) if imp else feature
    return imputed
//  [python]
def impute_values(feature, dt, sentinel):
    r"""Impute values for a given data type. The *median* strategy
    is applied for floating point values, and the *most frequent*
    strategy is applied for integer or Boolean values.

    Parameters
    ----------
    feature : pandas.Series or numpy.array
        The feature for imputation.
    dt : str
        The values ``'float64'``, ``'int64'``, or ``'bool'``.
    sentinel : float
        The number to be imputed for NaN values.

    Returns
    -------
    imputed : numpy.array
        The feature after imputation.

    Raises
    ------
    TypeError
        Data type ``dt`` is invalid for imputation.

    References
    ----------
    You can find more information on feature imputation here [IMP]_.

    .. [IMP] http://scikit-learn.org/stable/modules/preprocessing.html#imputation

    """

    try:
        # for pandas series
        feature = feature.values.reshape(-1, 1)
    except:
        # for numpy array
        feature = feature.reshape(-1, 1)

    if dt == 'float64':
        logger.info("    Imputation for Data Type %s: Median Strategy" % dt)
        # replace infinity with imputed value
        feature[np.isinf(feature)] = np.nan
        imp = SimpleImputer(missing_values=np.nan, strategy='median')
    elif dt == 'int64':
        logger.info("    Imputation for Data Type %s: Most Frequent Strategy" % dt)
        imp = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
    elif dt != 'bool':
        logger.info("    Imputation for Data Type %s: Fill Strategy with %d" % (dt, sentinel))
        imp = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=sentinel)
    else:
        logger.info("    No Imputation for Data Type %s" % dt)
        imp = None

    imputed = imp.fit_transform(feature) if imp else feature
    return imputed
//  [python]
def get_data_transformation_object(self):
        try:
            logging.info('Data Transformation initiated')
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']
            
            logging.info('Pipeline Initiated')

            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',Imputer(strategy='median')),
                ('scaler',StandardScaler())

                ]

            )

            # Categorigal Pipeline
            cat_pipeline=Pipeline(
                steps=[
                ('imputer',Imputer(strategy='most_frequent')),
                ('ordinalencoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]

            )

            preprocessor=ColumnTransformer([
            ('num_pipeline',num_pipeline,numerical_cols),
            ('cat_pipeline',cat_pipeline,categorical_cols)
            ])
            
            return preprocessor

            logging.info('Pipeline Completed')

        except Exception as e:
            logging.info("Error in Data Trnasformation")
            raise CustomException(e,sys)
//  [python]
def get_data_transformation_object(self):
        try:
            logging.info('Data Transformation initiated')
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']
            
            logging.info('Pipeline Initiated')

            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='median')),
                ('scaler',StandardScaler())

                ]

            )

            # Categorigal Pipeline
            cat_pipeline=Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='most_frequent')),
                ('ordinalencoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]

            )

            preprocessor=ColumnTransformer([
            ('num_pipeline',num_pipeline,numerical_cols),
            ('cat_pipeline',cat_pipeline,categorical_cols)
            ])
            
            return preprocessor

            logging.info('Pipeline Completed')

        except Exception as e:
            logging.info("Error in Data Trnasformation")
            raise CustomException(e,sys)
//  [python]
def _impute(self, matrix):
        imputer = s.Imputer(
            missing_values=self._missing_values,
            strategy=self._strategy,
            fill_value=self._fill_value,
            copy=self._copy,
        )
        imputed_matrix = imputer.fit_transform(matrix)
        return imputed_matrix
//  [python]
def _impute(self, matrix):
        imputer = _sklimpute.SimpleImputer(
            missing_values=self._missing_values,
            strategy=self._strategy,
            fill_value=self._fill_value,
            keep_empty_features=self._keep_empty_criteria,
        )
        imputed_matrix = imputer.fit_transform(matrix)
        return imputed_matrix
//  [python]
def test_pipeline(self):
        numeric_features = ["age", "fare"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["embarked", "sex", "pclass"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                ("classifier", LogisticRegression(solver="lbfgs")),
            ]
        )
        text = pipeline2str(clf)
        self.assertIn("StandardScaler", text)
        self.assertIn("Pipeline(embarked,sex,pclass)", text)
//  [python]
def test_pipeline(self):
        numeric_features = ["age", "fare"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["embarked", "sex", "pclass"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                ("classifier", LogisticRegression(solver="lbfgs")),
            ]
        )
        text = pipeline2str(clf)
        self.assertIn("StandardScaler", text)
        self.assertIn("Pipeline(embarked,sex,pclass)", text)
//  [python]
def simple_imputation(df_train, df_test, strategy="most_frequent"):
  """[summary]


  Args:
      df_train ([type]): [description]
      df_test ([type]): [description]
      strategy ([type]): [description]
      strategy supported
      "mean" : average
      "median" : med
      "most frequent" : mode
      Logic, input train and test. Append them and then do imputation

  Returns:
      [type]: [description]
  """
  ##imputer is fitted on both train and test. remove ".append(df_test)" to fit on only train
  df = df_train.append(df_test)
  imputer = Imputer(missing_values=np.nan, strategy=strategy)
  imputer = imputer.fit(df)
  col_list = df.columns
  df_train_imputed = pd.DataFrame(
    imputer.transform(df_train),
    columns=[str(col) + '_' + strategy for col in col_list])
  df_test_imputed = pd.DataFrame(
    imputer.transform(df_test),
    columns=[str(col) + '_' + strategy for col in col_list])
  return df_train_imputed, df_test_imputed
//  [python]
def simple_imputation(df_train, df_test, strategy="most_frequent"):
  """[summary]


  Args:
      df_train ([type]): [description]
      df_test ([type]): [description]
      strategy ([type]): [description]
      strategy supported
      "mean" : average
      "median" : med
      "most frequent" : mode
      Logic, input train and test. Append them and then do imputation

  Returns:
      [type]: [description]
  """
  ##imputer is fitted on both train and test. remove ".append(df_test)" to fit on only train
  df = df_train.append(df_test)
  imputer = SimpleImputer(missing_values=np.nan, strategy=strategy)
  imputer = imputer.fit(df)
  col_list = df.columns
  df_train_imputed = pd.DataFrame(
    imputer.transform(df_train),
    columns=[str(col) + '_' + strategy for col in col_list])
  df_test_imputed = pd.DataFrame(
    imputer.transform(df_test),
    columns=[str(col) + '_' + strategy for col in col_list])
  return df_train_imputed, df_test_imputed
//  [python]
lr = LogisticRegression(penalty="l1")
print(lr)

# %%
# Rich HTML representation
# ------------------------
# In notebooks estimators and pipelines will use a rich HTML representation.
# This is particularly useful to summarise the
# structure of pipelines and other composite estimators, with interactivity to
# provide detail.  Click on the example image below to expand Pipeline
# elements.  See :ref:`visualizing_composite_estimators` for how you can use
# this feature.

num_proc = make_pipeline(Imputer(strategy="median"), StandardScaler())
//  [python]
lr = LogisticRegression(penalty="l1")
print(lr)

# %%
# Rich HTML representation
# ------------------------
# In notebooks estimators and pipelines will use a rich HTML representation.
# This is particularly useful to summarise the
# structure of pipelines and other composite estimators, with interactivity to
# provide detail.  Click on the example image below to expand Pipeline
# elements.  See :ref:`visualizing_composite_estimators` for how you can use
# this feature.

num_proc = make_pipeline(SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def transform(self, X, y=None, **fit_params):
        from sklearn.preprocessing import Imputer

        X = X.to_numpy()
        imp = Imputer()
        X = imp.fit_transform(X)

        return self.fa.transform(X)
//  [python]
def transform(self, X, y=None, **fit_params):
        from sklearn.impute import SimpleImputer

        X = X.to_numpy()
        imp = SimpleImputer()
        X = imp.fit_transform(X)

        return self.fa.transform(X)
//  [python]
def fit_transform(self, X, y=None, **fit_params):
        from sklearn.decomposition import FactorAnalysis
        from sklearn.preprocessing import Imputer

        X = X.to_numpy()
        imp = Imputer()
        X = imp.fit_transform(X)
        n_components = self._n_components
        if min(X.shape) <= n_components:
            n_components = min(X.shape) - 1

        self.fa = FactorAnalysis(n_components=n_components)
        self.fa.fit(X)

        return self.fa.transform(X)
//  [python]
def fit_transform(self, X, y=None, **fit_params):
        from sklearn.decomposition import FactorAnalysis
        from sklearn.impute import SimpleImputer

        X = X.to_numpy()
        imp = SimpleImputer()
        X = imp.fit_transform(X)
        n_components = self._n_components
        if min(X.shape) <= n_components:
            n_components = min(X.shape) - 1

        self.fa = FactorAnalysis(n_components=n_components)
        self.fa.fit(X)

        return self.fa.transform(X)
//  [python]
def impute_categorical(
    categorical: pd.DataFrame,
    strategy: str,
    fill_value: str,
    custom_strategy: Optional[Callable[[pd.Series], object]] = None,
    **kwargs,
) -> pd.DataFrame:
    """Imputes missing values in categorical features.

    Parameters:
        df (pd.DataFrame): DataFrame with categorical features
        strategy (str, optional): The imputation strategy. Default is 'constant'.
        fill_value (str, optional): The value to use for imputation when strategy is 'constant'. Default is 'Missing'.
        custom_strategy (callable, optional): A custom function to compute the imputation value. Should take a Series and return an object.

    Returns:
        pd.DataFrame: DataFrame with imputed categorical features

    Example:
        To use a custom strategy that imputes missing values with the second most frequent value, you can do:

        def second_most_frequent(s):
            return s.value_counts().index[1] if len(s.value_counts()) > 1 else s.value_counts().index[0]

        impute_categorical(df, custom_strategy=second_most_frequent)
    """
    if strategy not in ["constant", "most_frequent"]:
        raise ValueError(f"Invalid strategy for categorical: {strategy}")

    if categorical.isnull().sum().sum() == 0:
        return categorical

    for col in categorical.columns:
        if custom_strategy:
            fill_value = custom_strategy(categorical[col])  # type: ignore
            categorical[col].fillna(fill_value, inplace=True)
        else:
            if strategy == "constant":
                categorical[col].fillna(fill_value, inplace=True)
            elif strategy == "most_frequent":
                categorical[col].fillna(categorical[col].value_counts().index[0], inplace=True)

    return categorical
//  [python]
def impute_categorical(
    categorical: pd.DataFrame,
    strategy: str,
    fill_value: str,
    custom_strategy: Optional[Callable[[pd.Series], object]] = None,
    **kwargs,
) -> pd.DataFrame:
    """Imputes missing values in categorical features.

    Parameters:
        df (pd.DataFrame): DataFrame with categorical features
        strategy (str, optional): The imputation strategy. Default is 'constant'.
        fill_value (str, optional): The value to use for imputation when strategy is 'constant'. Default is 'Missing'.
        custom_strategy (callable, optional): A custom function to compute the imputation value. Should take a Series and return an object.

    Returns:
        pd.DataFrame: DataFrame with imputed categorical features

    Example:
        To use a custom strategy that imputes missing values with the second most frequent value, you can do:

        def second_most_frequent(s):
            return s.value_counts().index[1] if len(s.value_counts()) > 1 else s.value_counts().index[0]

        impute_categorical(df, custom_strategy=second_most_frequent)
    """
    if strategy not in ["constant", "most_frequent"]:
        raise ValueError(f"Invalid strategy for categorical: {strategy}")

    if categorical.isnull().sum().sum() == 0:
        return categorical

    for col in categorical.columns:
        if custom_strategy:
            fill_value = custom_strategy(categorical[col])  # type: ignore
            categorical[col].fillna(fill_value, inplace=True)
        else:
            imputer = SimpleImputer(strategy=strategy, fill_value=fill_value, **kwargs)
            categorical[col] = imputer.fit_transform(categorical[[col]])[:, 0]

    return categorical