jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def get_preprocessor_object(self):
        try:
            numerical_features = ['age', 'bmi', 'children']
            categorical_features = ['sex', 'smoker', 'region']

            num_pipeline = Pipeline(
                steps = [
                    ("Imputer", Imputer(strategy="median")),
                    ("Scaler", StandardScaler(with_mean=False))
                ]
            )

            cat_pipeline = Pipeline(
                steps = [
                    ("Imputer", Imputer(strategy="most_frequent")),
                    ("OneHotEncoder", OneHotEncoder()),
                    ("Scaler", StandardScaler(with_mean=False))
                ]
            )

            logging.info("Handled missing values for both numerical and categorical features")
            logging.info("Numerical features are scaled")
            logging.info("Categorical features are encoded using OneHotEncoder")

            preprocessor = ColumnTransformer(
                [
                    ("num_pipeline", num_pipeline, numerical_features),
                    ("cat_pipeline", cat_pipeline, categorical_features)
                ]
            )

            return preprocessor

        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_preprocessor_object(self):
        try:
            numerical_features = ['age', 'bmi', 'children']
            categorical_features = ['sex', 'smoker', 'region']

            num_pipeline = Pipeline(
                steps = [
                    ("Imputer", SimpleImputer(strategy="median")),
                    ("Scaler", StandardScaler(with_mean=False))
                ]
            )

            cat_pipeline = Pipeline(
                steps = [
                    ("Imputer", SimpleImputer(strategy="most_frequent")),
                    ("OneHotEncoder", OneHotEncoder()),
                    ("Scaler", StandardScaler(with_mean=False))
                ]
            )

            logging.info("Handled missing values for both numerical and categorical features")
            logging.info("Numerical features are scaled")
            logging.info("Categorical features are encoded using OneHotEncoder")

            preprocessor = ColumnTransformer(
                [
                    ("num_pipeline", num_pipeline, numerical_features),
                    ("cat_pipeline", cat_pipeline, categorical_features)
                ]
            )

            return preprocessor

        except Exception as e:
            raise CustomException(e,sys)
//  [python]
machinefailuretype[num_cols].agg(['min','median','max']).T

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(machinefailuretype[num_cols + cat_cols],\
  machinefailuretype[['failtypecode']], test_size=0.2, random_state=0)


# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(3),Imputer(strategy="median"),
  MinMaxScaler())
//  [python]
machinefailuretype[num_cols].agg(['min','median','max']).T

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(machinefailuretype[num_cols + cat_cols],\
  machinefailuretype[['failtypecode']], test_size=0.2, random_state=0)


# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(3),SimpleImputer(strategy="median"),
  MinMaxScaler())
//  [python]
def test_simple_imputation_inverse_transform(missing_value):
    # Test inverse_transform feature for np.nan
    X_1 = np.array(
        [
            [9, missing_value, 3, -1],
            [4, -1, 5, 4],
            [6, 7, missing_value, -1],
            [8, 9, 0, missing_value],
        ]
    )

    X_2 = np.array(
        [
            [5, 4, 2, 1],
            [2, 1, missing_value, 3],
            [9, missing_value, 7, 1],
            [6, 4, 2, missing_value],
        ]
    )

    X_3 = np.array(
        [
            [1, missing_value, 5, 9],
            [missing_value, 4, missing_value, missing_value],
            [2, missing_value, 7, missing_value],
            [missing_value, 3, missing_value, 8],
        ]
    )

    X_4 = np.array(
        [
            [1, 1, 1, 3],
            [missing_value, 2, missing_value, 1],
            [2, 3, 3, 4],
            [missing_value, 4, missing_value, 2],
        ]
    )

    imputer = Imputer(
        missing_values=missing_value, strategy="mean", add_indicator=True
    )

    X_1_trans = imputer.fit_transform(X_1)
    X_1_inv_trans = imputer.inverse_transform(X_1_trans)

    X_2_trans = imputer.transform(X_2)  # test on new data
    X_2_inv_trans = imputer.inverse_transform(X_2_trans)

    assert_array_equal(X_1_inv_trans, X_1)
    assert_array_equal(X_2_inv_trans, X_2)

    for X in [X_3, X_4]:
        X_trans = imputer.fit_transform(X)
        X_inv_trans = imputer.inverse_transform(X_trans)
        assert_array_equal(X_inv_trans, X)
//  [python]
def test_simple_imputation_inverse_transform(missing_value):
    # Test inverse_transform feature for np.nan
    X_1 = np.array(
        [
            [9, missing_value, 3, -1],
            [4, -1, 5, 4],
            [6, 7, missing_value, -1],
            [8, 9, 0, missing_value],
        ]
    )

    X_2 = np.array(
        [
            [5, 4, 2, 1],
            [2, 1, missing_value, 3],
            [9, missing_value, 7, 1],
            [6, 4, 2, missing_value],
        ]
    )

    X_3 = np.array(
        [
            [1, missing_value, 5, 9],
            [missing_value, 4, missing_value, missing_value],
            [2, missing_value, 7, missing_value],
            [missing_value, 3, missing_value, 8],
        ]
    )

    X_4 = np.array(
        [
            [1, 1, 1, 3],
            [missing_value, 2, missing_value, 1],
            [2, 3, 3, 4],
            [missing_value, 4, missing_value, 2],
        ]
    )

    imputer = SimpleImputer(
        missing_values=missing_value, strategy="mean", add_indicator=True
    )

    X_1_trans = imputer.fit_transform(X_1)
    X_1_inv_trans = imputer.inverse_transform(X_1_trans)

    X_2_trans = imputer.transform(X_2)  # test on new data
    X_2_inv_trans = imputer.inverse_transform(X_2_trans)

    assert_array_equal(X_1_inv_trans, X_1)
    assert_array_equal(X_2_inv_trans, X_2)

    for X in [X_3, X_4]:
        X_trans = imputer.fit_transform(X)
        X_inv_trans = imputer.inverse_transform(X_trans)
        assert_array_equal(X_inv_trans, X)
//  [python]
def _init_preprocessor(self):
        # preprocessing pipeline
        self._scaler = StandardScaler()  # recenter
        self._imputer = Imputer(strategy="mean")  # mean should be 0 anyway
        self._forest = RandomForestClassifier(n_estimators=100, n_jobs=-1)
        self._selector = SelectPercentile(
            score_func=self._forest_score, percentile=100 * self.importance_percentile
        )

        # label encoder for multiclass
        self._label_encoder = LabelEncoder()
//  [python]
def _init_preprocessor(self):
        # preprocessing pipeline
        self._scaler = StandardScaler()  # recenter
        self._imputer = SimpleImputer(strategy="mean")  # mean should be 0 anyway
        self._forest = RandomForestClassifier(n_estimators=100, n_jobs=-1)
        self._selector = SelectPercentile(
            score_func=self._forest_score, percentile=100 * self.importance_percentile
        )

        # label encoder for multiclass
        self._label_encoder = LabelEncoder()
//  [python]
def test_pipeline_compatibility(strategy: str) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
        }
    )
    y = pd.Series([0, 1, 2, 0, 1, 0])
    numeric_preprocessor = Pipeline(
        [
            ("imputer", Imputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(preprocessor, LogisticRegression())
    pipe.fit(X, y)
    mapie = MapieClassifier(estimator=pipe, **STRATEGIES[strategy][0])
    mapie.fit(X, y)
    mapie.predict(X)
//  [python]
def test_pipeline_compatibility(strategy: str) -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
        }
    )
    y = pd.Series([0, 1, 2, 0, 1, 0])
    numeric_preprocessor = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[
            ("encoding", OneHotEncoder(handle_unknown="ignore"))
        ]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"])
        ]
    )
    pipe = make_pipeline(preprocessor, LogisticRegression())
    pipe.fit(X, y)
    mapie = MapieClassifier(estimator=pipe, **STRATEGIES[strategy][0])
    mapie.fit(X, y)
    mapie.predict(X)
//  [python]
def fit(self, X, y=None, **fit_params):
        # # Feature Transformation based on different types of features
        # types = type_detection(X)
        # num_cols = []
        # bool_cols = []
        # cat_cols = []
        # for i, t in enumerate(types):
        #     if t == NumericalFeature:
        #         num_cols.append(i)
        #     elif t == BooleanFeature:
        #         bool_cols.append(i)
        #     else:
        #         cat_cols.append(i)

        encoder = {
            'TargetEncoder': TargetEncoder(),
            'LeaveOneOutEncoder': LeaveOneOutEncoder(),
            'BinaryEncoder': BinaryEncoder(),
            'CatBoostEncoder': CatBoostEncoder(),
            'PolynomialEncoder': PolynomialEncoder(),
            'BackwardDifferenceEncoder': BackwardDifferenceEncoder(),
            'CountEncoder': CountEncoder(),
            'HashingEncoder': HashingEncoder(),
            'HelmertEncoder': HelmertEncoder(),
            'JamesSteinEncoder': JamesSteinEncoder(),
            'MEstimateEncoder': MEstimateEncoder(),
            'OneHotEncoder': OneHotEncoder(),
            'OrdinalEncoder': OrdinalEncoder(),
            'QuantileEncoder': QuantileEncoder(),
            'SumEncoder': SumEncoder(),
            'SummaryEncoder': SummaryEncoder(),
            # 'GLMMEncoder': GLMMEncoder(), # Unavailable
        }[self.encoding_scheme]
        num_transformer = Pipeline([
            ('Imputer', Imputer(strategy='mean')),
            ('Encoder', StandardScaler()),
        ])
        cat_transformer = Pipeline(
            [('Imputer', Imputer(strategy='constant', fill_value='missing')),
             ('Encoder', encoder),
             ('Scaler', StandardScaler())]
        )
        transformer = ColumnTransformer(
            transformers=[
                ('num', num_transformer, selector(dtype_include='float64')),
                ('cat', cat_transformer, selector(dtype_include='category')),
                ('obj', cat_transformer, selector(dtype_include='object')),
            ])

        self.x_scaler = transformer
        self.x_scaler.fit(X, y)
        return self
//  [python]
def fit(self, X, y=None, **fit_params):
        # # Feature Transformation based on different types of features
        # types = type_detection(X)
        # num_cols = []
        # bool_cols = []
        # cat_cols = []
        # for i, t in enumerate(types):
        #     if t == NumericalFeature:
        #         num_cols.append(i)
        #     elif t == BooleanFeature:
        #         bool_cols.append(i)
        #     else:
        #         cat_cols.append(i)

        encoder = {
            'TargetEncoder': TargetEncoder(),
            'LeaveOneOutEncoder': LeaveOneOutEncoder(),
            'BinaryEncoder': BinaryEncoder(),
            'CatBoostEncoder': CatBoostEncoder(),
            'PolynomialEncoder': PolynomialEncoder(),
            'BackwardDifferenceEncoder': BackwardDifferenceEncoder(),
            'CountEncoder': CountEncoder(),
            'HashingEncoder': HashingEncoder(),
            'HelmertEncoder': HelmertEncoder(),
            'JamesSteinEncoder': JamesSteinEncoder(),
            'MEstimateEncoder': MEstimateEncoder(),
            'OneHotEncoder': OneHotEncoder(),
            'OrdinalEncoder': OrdinalEncoder(),
            'QuantileEncoder': QuantileEncoder(),
            'SumEncoder': SumEncoder(),
            'SummaryEncoder': SummaryEncoder(),
            # 'GLMMEncoder': GLMMEncoder(), # Unavailable
        }[self.encoding_scheme]
        num_transformer = Pipeline([
            ('Imputer', SimpleImputer(strategy='mean')),
            ('Encoder', StandardScaler()),
        ])
        cat_transformer = Pipeline(
            [('Imputer', SimpleImputer(strategy='constant', fill_value='missing')),
             ('Encoder', encoder),
             ('Scaler', StandardScaler())]
        )
        transformer = ColumnTransformer(
            transformers=[
                ('num', num_transformer, selector(dtype_include='float64')),
                ('cat', cat_transformer, selector(dtype_include='category')),
                ('obj', cat_transformer, selector(dtype_include='object')),
            ])

        self.x_scaler = transformer
        self.x_scaler.fit(X, y)
        return self
//  [python]
def __init__(
        self,
        config: Optional[CFG] = None,
        **kwargs: Any,
    ) -> None:
        """

        Parameters:
        -----------
        config: CFG, optional,
            configurations, ref. `cfg.OutcomeCfg`

        """
        self.config = deepcopy(config)
        self.__imputer = Imputer(missing_values=np.nan, strategy="mean")
        self.__scaler = StandardScaler()

        self.logger_manager = None
        self.reader = None
        self.__df_features = None
        self.x_train, self.y_train = None, None
        self.x_test, self.y_test = None, None
        self._prepare_training_data()

        self.__cache = {}
        self.best_clf, self.best_params, self.best_score = None, None, None
        self._no = 1

        self._num_workers = max(1, mp.cpu_count() - 2)
//  [python]
def __init__(
        self,
        config: Optional[CFG] = None,
        **kwargs: Any,
    ) -> None:
        """

        Parameters:
        -----------
        config: CFG, optional,
            configurations, ref. `cfg.OutcomeCfg`

        """
        self.config = deepcopy(config)
        self.__imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
        self.__scaler = StandardScaler()

        self.logger_manager = None
        self.reader = None
        self.__df_features = None
        self.x_train, self.y_train = None, None
        self.x_test, self.y_test = None, None
        self._prepare_training_data()

        self.__cache = {}
        self.best_clf, self.best_params, self.best_score = None, None, None
        self._no = 1

        self._num_workers = max(1, mp.cpu_count() - 2)
//  [python]
def test_support_missing_values(MultiOutputEstimator, Estimator):
    # smoke test to check that pipeline MultioutputEstimators are letting
    # the validation of missing values to
    # the underlying pipeline, regressor or classifier
    rng = np.random.RandomState(42)
    X, y = rng.randn(50, 2), rng.binomial(1, 0.5, (50, 3))
    mask = rng.choice([1, 0], X.shape, p=[0.01, 0.99]).astype(bool)
    X[mask] = np.nan

    pipe = make_pipeline(Imputer(), Estimator())
    MultiOutputEstimator(pipe).fit(X, y).score(X, y)
//  [python]
def test_support_missing_values(MultiOutputEstimator, Estimator):
    # smoke test to check that pipeline MultioutputEstimators are letting
    # the validation of missing values to
    # the underlying pipeline, regressor or classifier
    rng = np.random.RandomState(42)
    X, y = rng.randn(50, 2), rng.binomial(1, 0.5, (50, 3))
    mask = rng.choice([1, 0], X.shape, p=[0.01, 0.99]).astype(bool)
    X[mask] = np.nan

    pipe = make_pipeline(SimpleImputer(), Estimator())
    MultiOutputEstimator(pipe).fit(X, y).score(X, y)
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with SimpleImputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values=np.nan)),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with SimpleImputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_mapper(self):
		domain = ContinuousDomain(with_statistics = True)
		X = DataFrame([[2.0, 2], [1.0, 0.5], [2, float("NaN")], [float("NaN"), 2], [2.0, float("NaN")], [3.0, 3.5]], columns = ["x1", "x2"])
		mapper = DataFrameMapper([
			(["x1", "x2"], [domain, Imputer(), StandardScaler()]),
		], input_df = True, df_out = True)
		Xt = mapper.fit_transform(X)
		self.assertIsInstance(Xt, DataFrame)
		self.assertEqual(2, domain.n_features_in_)
		self.assertEqual(["x1", "x2"], domain.feature_names_in_.tolist())
		self.assertEqual({"totalFreq" : [6, 6], "missingFreq" : [1, 2], "invalidFreq" : [0, 0]}, _array_to_list(domain.counts_))
		self.assertEqual({"minimum" : [1.0, 0.5], "maximum" : [3.0, 3.5], "mean" : [2.0, 2.0]}, _array_to_list(dict((k, domain.numeric_info_[k]) for k in ["minimum", "maximum", "mean"])))
		self.assertEqual([1.0, 0.5], domain.data_min_.tolist())
		self.assertEqual([3.0, 3.5], domain.data_max_.tolist())
//  [python]
def test_mapper(self):
		domain = ContinuousDomain(with_statistics = True)
		X = DataFrame([[2.0, 2], [1.0, 0.5], [2, float("NaN")], [float("NaN"), 2], [2.0, float("NaN")], [3.0, 3.5]], columns = ["x1", "x2"])
		mapper = DataFrameMapper([
			(["x1", "x2"], [domain, SimpleImputer(), StandardScaler()]),
		], input_df = True, df_out = True)
		Xt = mapper.fit_transform(X)
		self.assertIsInstance(Xt, DataFrame)
		self.assertEqual(2, domain.n_features_in_)
		self.assertEqual(["x1", "x2"], domain.feature_names_in_.tolist())
		self.assertEqual({"totalFreq" : [6, 6], "missingFreq" : [1, 2], "invalidFreq" : [0, 0]}, _array_to_list(domain.counts_))
		self.assertEqual({"minimum" : [1.0, 0.5], "maximum" : [3.0, 3.5], "mean" : [2.0, 2.0]}, _array_to_list(dict((k, domain.numeric_info_[k]) for k in ["minimum", "maximum", "mean"])))
		self.assertEqual([1.0, 0.5], domain.data_min_.tolist())
		self.assertEqual([3.0, 3.5], domain.data_max_.tolist())
//  [python]
def test_pipeline_with_nans(ClsRFE):
    """Check that RFE works with pipeline that accept nans.

    Non-regression test for gh-21743.
    """
    X, y = load_iris(return_X_y=True)
    X[0, 0] = np.nan

    pipe = make_pipeline(
        Imputer(),
        StandardScaler(),
        LogisticRegression(),
    )

    fs = ClsRFE(
        estimator=pipe,
        importance_getter="named_steps.logisticregression.coef_",
    )
    fs.fit(X, y)
//  [python]
def test_pipeline_with_nans(ClsRFE):
    """Check that RFE works with pipeline that accept nans.

    Non-regression test for gh-21743.
    """
    X, y = load_iris(return_X_y=True)
    X[0, 0] = np.nan

    pipe = make_pipeline(
        SimpleImputer(),
        StandardScaler(),
        LogisticRegression(),
    )

    fs = ClsRFE(
        estimator=pipe,
        importance_getter="named_steps.logisticregression.coef_",
    )
    fs.fit(X, y)
//  [python]
categorical_encoder = OrdinalEncoder(
    handle_unknown="use_encoded_value", unknown_value=-1, encoded_missing_value=-1
)
numerical_pipe = Imputer(strategy="mean")
//  [python]
categorical_encoder = OrdinalEncoder(
    handle_unknown="use_encoded_value", unknown_value=-1, encoded_missing_value=-1
)
numerical_pipe = SimpleImputer(strategy="mean")
//  [python]
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", Imputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)
//  [python]
numeric_selector = make_column_selector(dtype_include=np.number)
categorical_selector = make_column_selector(dtype_include=object)


numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)
//  [python]
def __init__(
        self, features: list, strategy: Literal["mean", "median", "most_frequent", "constant"] = "mean", fill_value=None
    ):
        """
        Initialize self.

        Args:
            features (list): Columns to be processed.
            strategy (Literal["mean", "median", "most_frequent", "constant"], optional): The imputation strategy, notice 'mean' and 'median' can only
                                      be used for numeric features. Defaults to 'mean'.
            fill_value (int, optional): Fill_value is used to replace all occurrences of missing_values.
                                        Defaults to None.
        """
        self.features = features
        self.model = Imputer(strategy=strategy, fill_value=fill_value)
//  [python]
def __init__(
        self, features: list, strategy: Literal["mean", "median", "most_frequent", "constant"] = "mean", fill_value=None
    ):
        """
        Initialize self.

        Args:
            features (list): Columns to be processed.
            strategy (Literal["mean", "median", "most_frequent", "constant"], optional): The imputation strategy, notice 'mean' and 'median' can only
                                      be used for numeric features. Defaults to 'mean'.
            fill_value (int, optional): Fill_value is used to replace all occurrences of missing_values.
                                        Defaults to None.
        """
        self.features = features
        self.model = SimpleImputer(strategy=strategy, fill_value=fill_value)
//  [python]
def fit(self, X, y=None):
        """Fit the imputer on X.

        Parameters
        ----------
        X : {array-like}, shape (n_samples, n_features)
            Input data, where ``n_samples`` is the number of samples and
            ``n_features`` is the number of features.

        Returns
        -------
        self : RobustImputer
        """
        X = self._validate_input(X)

        self.vectorized_mask_function_ = self.mask_function or is_finite_numeric
        X = _apply_mask(X, _get_mask(X, self.vectorized_mask_function_))

        if self._is_constant_multicolumn_imputation():
            self.simple_imputer_ = Imputer(strategy=self.strategy)
        else:
            self.simple_imputer_ = Imputer(strategy=self.strategy, fill_value=self.fill_values)

        self.simple_imputer_.fit(X)

        # set "SimpleImputer.statistics_" for multicolumn imputations with different column fill values
        # SimpleImputer cannot preform multicolumn imputation with different column fill values
        if self._is_constant_multicolumn_imputation():
            self.simple_imputer_.statistics_ = np.asarray(self.fill_values)

        return self
//  [python]
def fit(self, X, y=None):
        """Fit the imputer on X.

        Parameters
        ----------
        X : {array-like}, shape (n_samples, n_features)
            Input data, where ``n_samples`` is the number of samples and
            ``n_features`` is the number of features.

        Returns
        -------
        self : RobustImputer
        """
        X = self._validate_input(X)

        self.vectorized_mask_function_ = self.mask_function or is_finite_numeric
        X = _apply_mask(X, _get_mask(X, self.vectorized_mask_function_))

        if self._is_constant_multicolumn_imputation():
            self.simple_imputer_ = SimpleImputer(strategy=self.strategy)
        else:
            self.simple_imputer_ = SimpleImputer(strategy=self.strategy, fill_value=self.fill_values)

        self.simple_imputer_.fit(X)

        # set "SimpleImputer.statistics_" for multicolumn imputations with different column fill values
        # SimpleImputer cannot preform multicolumn imputation with different column fill values
        if self._is_constant_multicolumn_imputation():
            self.simple_imputer_.statistics_ = np.asarray(self.fill_values)

        return self
//  [python]
def test_grid_search_allows_nans():
    # Test dcv.GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]

    imputer = Imputer(strategy="mean", missing_values=np.nan)
    p = Pipeline([("imputer", imputer), ("classifier", MockClassifier())])
    dcv.GridSearchCV(p, {"classifier__foo_param": [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_grid_search_allows_nans():
    # Test dcv.GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]

    imputer = SimpleImputer(strategy="mean", missing_values=np.nan)
    p = Pipeline([("imputer", imputer), ("classifier", MockClassifier())])
    dcv.GridSearchCV(p, {"classifier__foo_param": [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_support_missing_values(MultiClassClassifier):
    # smoke test to check that pipeline OvR and OvO classifiers are letting
    # the validation of missing values to
    # the underlying pipeline or classifiers
    rng = np.random.RandomState(42)
    X, y = iris.data, iris.target
    X = np.copy(X)  # Copy to avoid that the original data is modified
    mask = rng.choice([1, 0], X.shape, p=[.1, .9]).astype(bool)
    X[mask] = np.nan
    lr = make_pipeline(Imputer(),
                       LogisticRegression(random_state=rng))

    MultiClassClassifier(lr).fit(X, y).score(X, y)
//  [python]
def test_support_missing_values(MultiClassClassifier):
    # smoke test to check that pipeline OvR and OvO classifiers are letting
    # the validation of missing values to
    # the underlying pipeline or classifiers
    rng = np.random.RandomState(42)
    X, y = iris.data, iris.target
    X = np.copy(X)  # Copy to avoid that the original data is modified
    mask = rng.choice([1, 0], X.shape, p=[.1, .9]).astype(bool)
    X[mask] = np.nan
    lr = make_pipeline(SimpleImputer(),
                       LogisticRegression(random_state=rng))

    MultiClassClassifier(lr).fit(X, y).score(X, y)
//  [python]
def fill_na(train: pd.DataFrame, parameters: Dict[str, Any]) -> pd.DataFrame:
    train[parameters['none_cols']] = train[parameters['none_cols']].fillna("None")
    train[parameters['zero_cols']] = train[parameters['zero_cols']].fillna(0)

    train["LotFrontage"] = train.groupby("Neighborhood")["LotFrontage"].transform(
                                        lambda x: x.fillna(x.median()))

    impute_int = Imputer(missing_values=np.nan, strategy='most_frequent')
    impute_str = Imputer(missing_values=np.nan, strategy='most_frequent')

    int_cols = train.select_dtypes(include='number').columns
    str_cols = train.select_dtypes(exclude='number').columns

    train[int_cols] = impute_int.fit_transform(train[int_cols])
    train[str_cols] = impute_str.fit_transform(train[str_cols])

    return train
//  [python]
def fill_na(train: pd.DataFrame, parameters: Dict[str, Any]) -> pd.DataFrame:
    train[parameters['none_cols']] = train[parameters['none_cols']].fillna("None")
    train[parameters['zero_cols']] = train[parameters['zero_cols']].fillna(0)

    train["LotFrontage"] = train.groupby("Neighborhood")["LotFrontage"].transform(
                                        lambda x: x.fillna(x.median()))

    impute_int = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
    impute_str = SimpleImputer(missing_values=np.nan, strategy='most_frequent')

    int_cols = train.select_dtypes(include='number').columns
    str_cols = train.select_dtypes(exclude='number').columns

    train[int_cols] = impute_int.fit_transform(train[int_cols])
    train[str_cols] = impute_str.fit_transform(train[str_cols])

    return train
//  [python]
def _restore_model(self):
        """
        restore inner keras model after running make_picklable
        """

        model_json, model_weights = self.model
        self.model = tf.keras.models.model_from_json(model_json)
        self.model.set_weights(model_weights)
        if not hasattr(self, "_scale_impute"):
            self.xscale = "minmax"
            self._set_scale_impute(
                impute_missing=-1,
                xscale_before_impute=True,
                scaler=self._scaler,
                imputer=Imputer(
                    missing_values=np.nan,
                    strategy="constant",
                    fill_value=-1,
                ).fit(np.zeros((1, self.n_feat))),
            )
//  [python]
def _restore_model(self):
        """
        restore inner keras model after running make_picklable
        """

        model_json, model_weights = self.model
        self.model = tf.keras.models.model_from_json(model_json)
        self.model.set_weights(model_weights)
        if not hasattr(self, "_scale_impute"):
            self.xscale = "minmax"
            self._set_scale_impute(
                impute_missing=-1,
                xscale_before_impute=True,
                scaler=self._scaler,
                imputer=SimpleImputer(
                    missing_values=np.nan,
                    strategy="constant",
                    fill_value=-1,
                ).fit(np.zeros((1, self.n_feat))),
            )
//  [python]
def test_imputer(
    failure_logger,
    random_seed,
    int_dataset,  # noqa: F811
    strategy,
    missing_values,
    add_indicator,
):
    zero_filled, one_filled, nan_filled = int_dataset
    if missing_values == 0:
        X_np, X = zero_filled
    elif missing_values == 1:
        X_np, X = one_filled
    else:
        X_np, X = nan_filled
    np.random.seed(random_seed)
    fill_value = np.random.randint(10, size=1)[0]

    imputer = cuSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
        add_indicator=add_indicator,
    )
    t_X = imputer.fit_transform(X)
    assert type(t_X) == type(X)

    imputer = s.Imputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
        add_indicator=add_indicator,
    )
    sk_t_X = imputer.fit_transform(X_np)

    assert_allclose(t_X, sk_t_X)
//  [python]
def test_imputer(
    failure_logger,
    random_seed,
    int_dataset,  # noqa: F811
    strategy,
    missing_values,
    add_indicator,
):
    zero_filled, one_filled, nan_filled = int_dataset
    if missing_values == 0:
        X_np, X = zero_filled
    elif missing_values == 1:
        X_np, X = one_filled
    else:
        X_np, X = nan_filled
    np.random.seed(random_seed)
    fill_value = np.random.randint(10, size=1)[0]

    imputer = cuSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
        add_indicator=add_indicator,
    )
    t_X = imputer.fit_transform(X)
    assert type(t_X) == type(X)

    imputer = skSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
        add_indicator=add_indicator,
    )
    sk_t_X = imputer.fit_transform(X_np)

    assert_allclose(t_X, sk_t_X)
//  [python]
def test_imputer_sparse(sparse_imputer_dataset, strategy):  # noqa: F811
    missing_values, X_sp, X = sparse_imputer_dataset

    if X.format == "csr":
        pytest.skip("Skipping CSR matrices")

    fill_value = np.random.randint(10, size=1)[0]

    imputer = cuSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
    )
    t_X = imputer.fit_transform(X)
    #  assert type(t_X) == type(X)
    if cpx.scipy.sparse.issparse(X):
        assert cpx.scipy.sparse.issparse(t_X)
    if scipy.sparse.issparse(X):
        assert scipy.sparse.issparse(t_X)

    imputer = Imputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
    )
    sk_t_X = imputer.fit_transform(X_sp)
    assert_allclose(t_X, sk_t_X)
//  [python]
def test_imputer_sparse(sparse_imputer_dataset, strategy):  # noqa: F811
    missing_values, X_sp, X = sparse_imputer_dataset

    if X.format == "csr":
        pytest.skip("Skipping CSR matrices")

    fill_value = np.random.randint(10, size=1)[0]

    imputer = cuSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
    )
    t_X = imputer.fit_transform(X)
    #  assert type(t_X) == type(X)
    if cpx.scipy.sparse.issparse(X):
        assert cpx.scipy.sparse.issparse(t_X)
    if scipy.sparse.issparse(X):
        assert scipy.sparse.issparse(t_X)

    imputer = skSimpleImputer(
        copy=True,
        missing_values=missing_values,
        strategy=strategy,
        fill_value=fill_value,
    )
    sk_t_X = imputer.fit_transform(X_sp)
    assert_allclose(t_X, sk_t_X)
//  [python]
def get_data_transformer_object(self):
        '''
        This function si responsible for data trnasformation
        
        '''
        try:
            numerical_columns = ["writing_score", "reading_score"]
            categorical_columns = [
                "gender",
                "race_ethnicity",
                "parental_level_of_education",
                "lunch",
                "test_preparation_course",
            ]

            num_pipeline= Pipeline(
                steps=[
                ("imputer",Imputer(strategy="median")),
                ("scaler",StandardScaler())

                ]
            )

            cat_pipeline=Pipeline(

                steps=[
                ("imputer",Imputer(strategy="most_frequent")),
                ("one_hot_encoder",OneHotEncoder()),
                ("scaler",StandardScaler(with_mean=False))
                ]

            )

            logging.info(f"Categorical columns: {categorical_columns}")
            logging.info(f"Numerical columns: {numerical_columns}")

            preprocessor=ColumnTransformer(
                [
                ("num_pipeline",num_pipeline,numerical_columns),
                ("cat_pipelines",cat_pipeline,categorical_columns)

                ]


            )

            return preprocessor
        
        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_data_transformer_object(self):
        '''
        This function si responsible for data trnasformation
        
        '''
        try:
            numerical_columns = ["writing_score", "reading_score"]
            categorical_columns = [
                "gender",
                "race_ethnicity",
                "parental_level_of_education",
                "lunch",
                "test_preparation_course",
            ]

            num_pipeline= Pipeline(
                steps=[
                ("imputer",SimpleImputer(strategy="median")),
                ("scaler",StandardScaler())

                ]
            )

            cat_pipeline=Pipeline(

                steps=[
                ("imputer",SimpleImputer(strategy="most_frequent")),
                ("one_hot_encoder",OneHotEncoder()),
                ("scaler",StandardScaler(with_mean=False))
                ]

            )

            logging.info(f"Categorical columns: {categorical_columns}")
            logging.info(f"Numerical columns: {numerical_columns}")

            preprocessor=ColumnTransformer(
                [
                ("num_pipeline",num_pipeline,numerical_columns),
                ("cat_pipelines",cat_pipeline,categorical_columns)

                ]


            )

            return preprocessor
        
        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def _set_scale_impute(
        self, impute_missing, xscale_before_impute, scaler=None, imputer=None
    ):
        """
        Sets the inner scaling and imputer mechanism.
        impute_missing: Determines how the NaN features are treated.
                If str, defines the strategy used in the scikit-learn SimpleImputer,
                e.g., "mean" sets the NaNs to the mean of their feature column.
                If a float is provided, this float is used to replace NaNs in the original dataset.
        xscale_before_impute: whether to first scale the input and then impute values, or
                first impute values and then scale the inputs.
        scaler: optional sklearn scaler to use
        imputer: optional sklearn imputer to use
        """
        # Define the scaler
        if scaler is not None:
            self._scaler = scaler
        elif self.xscale == "minmax":
            self._scaler = MinMaxScaler(feature_range=(-0.5, 0.5))

        elif self.xscale == "standard":
            self._scaler = StandardScaler()

        # Define the imputer
        if imputer is not None:
            self._imputer = imputer
        elif isinstance(impute_missing, str):
            self._imputer = Imputer(
                missing_values=np.nan, strategy=impute_missing
            )
        else:
            self._imputer = Imputer(
                missing_values=np.nan, strategy="constant", fill_value=impute_missing
            )

        # Scale and impute input features in the desired order
        if xscale_before_impute:
            self._scale_impute = Pipeline(
                [("scaler", self._scaler), ("imputer", self._imputer)]
            )
        else:
            self._scale_impute = Pipeline(
                [("imputer", self._imputer), ("scaler", self._scaler)]
            )
//  [python]
def _set_scale_impute(
        self, impute_missing, xscale_before_impute, scaler=None, imputer=None
    ):
        """
        Sets the inner scaling and imputer mechanism.
        impute_missing: Determines how the NaN features are treated.
                If str, defines the strategy used in the scikit-learn SimpleImputer,
                e.g., "mean" sets the NaNs to the mean of their feature column.
                If a float is provided, this float is used to replace NaNs in the original dataset.
        xscale_before_impute: whether to first scale the input and then impute values, or
                first impute values and then scale the inputs.
        scaler: optional sklearn scaler to use
        imputer: optional sklearn imputer to use
        """
        # Define the scaler
        if scaler is not None:
            self._scaler = scaler
        elif self.xscale == "minmax":
            self._scaler = MinMaxScaler(feature_range=(-0.5, 0.5))

        elif self.xscale == "standard":
            self._scaler = StandardScaler()

        # Define the imputer
        if imputer is not None:
            self._imputer = imputer
        elif isinstance(impute_missing, str):
            self._imputer = SimpleImputer(
                missing_values=np.nan, strategy=impute_missing
            )
        else:
            self._imputer = SimpleImputer(
                missing_values=np.nan, strategy="constant", fill_value=impute_missing
            )

        # Scale and impute input features in the desired order
        if xscale_before_impute:
            self._scale_impute = Pipeline(
                [("scaler", self._scaler), ("imputer", self._imputer)]
            )
        else:
            self._scale_impute = Pipeline(
                [("imputer", self._imputer), ("scaler", self._scaler)]
            )
//  [python]
def get_data_transformer_object(self):
        '''
        this function is responsible for data transformation
        '''
        try:
            numerical_columns = ["writing_score", "reading_score"]
            categorical_columns = [
                "gender",
                "race_ethnicity",
                "parental_level_of_education",
                "lunch",
                "test_preparation_course",
            ]
            num_pipeline=Pipeline(steps=[
                ("imputer",Imputer(strategy='median')),
                ('scalar',StandardScaler())

            ])
            cat_pipeline=Pipeline(steps=[
            ("imputer",Imputer(strategy="most_frequent")),
            ("one_hot_encoder",OneHotEncoder()),
            ("scaler",StandardScaler(with_mean=False))
            ])

            logging.info(f"Categorical Columns:{categorical_columns}")
            logging.info(f"Numerical Columns:{numerical_columns}")

            preprocessor=ColumnTransformer(
                [
                    ("num_pipeline",num_pipeline,numerical_columns),
                    ("cat_pipeline",cat_pipeline,categorical_columns)
                ]

            )
            return preprocessor
            

        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_data_transformer_object(self):
        '''
        this function is responsible for data transformation
        '''
        try:
            numerical_columns = ["writing_score", "reading_score"]
            categorical_columns = [
                "gender",
                "race_ethnicity",
                "parental_level_of_education",
                "lunch",
                "test_preparation_course",
            ]
            num_pipeline=Pipeline(steps=[
                ("imputer",SimpleImputer(strategy='median')),
                ('scalar',StandardScaler())

            ])
            cat_pipeline=Pipeline(steps=[
            ("imputer",SimpleImputer(strategy="most_frequent")),
            ("one_hot_encoder",OneHotEncoder()),
            ("scaler",StandardScaler(with_mean=False))
            ])

            logging.info(f"Categorical Columns:{categorical_columns}")
            logging.info(f"Numerical Columns:{numerical_columns}")

            preprocessor=ColumnTransformer(
                [
                    ("num_pipeline",num_pipeline,numerical_columns),
                    ("cat_pipeline",cat_pipeline,categorical_columns)
                ]

            )
            return preprocessor
            

        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_preprocessing_steps(
            categorical_features,
            missing_values_per_feature,
            numeric_transformer,
            numeric_features,
            sparse
    ):
        if not isinstance(categorical_features, collections.abc.Iterable):
            raise ValueError(f"categorical_features must be iterable but is {type(categorical_features)}")
        if not isinstance(missing_values_per_feature, collections.abc.Iterable):
            raise ValueError(f"missing_values_per_feature must be iterable but is {type(missing_values_per_feature)}")
        if not isinstance(sparse, bool):
            raise ValueError(f"`sparse` must be a bool but is {type(sparse)}")
        if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
            categorical_transformer = Pipeline([
                ("imputer", Imputer(strategy="most_frequent")),
                ("binarizer", OneHotEncoder(handle_unknown='ignore', sparse_output=sparse)),

            ])
            return [("impute_and_binarize", ColumnTransformer(
                transformers=[
                    ("num", numeric_transformer, numeric_features),
                    ("cat", categorical_transformer, categorical_features),
                ]
            ))]
        else:
            return []
//  [python]
def get_preprocessing_steps(
            categorical_features,
            missing_values_per_feature,
            numeric_transformer,
            numeric_features,
            sparse
    ):
        if not isinstance(categorical_features, collections.abc.Iterable):
            raise ValueError(f"categorical_features must be iterable but is {type(categorical_features)}")
        if not isinstance(missing_values_per_feature, collections.abc.Iterable):
            raise ValueError(f"missing_values_per_feature must be iterable but is {type(missing_values_per_feature)}")
        if not isinstance(sparse, bool):
            raise ValueError(f"`sparse` must be a bool but is {type(sparse)}")
        if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
            categorical_transformer = Pipeline([
                ("imputer", SimpleImputer(strategy="most_frequent")),
                ("binarizer", OneHotEncoder(handle_unknown='ignore', sparse_output=sparse)),

            ])
            return [("impute_and_binarize", ColumnTransformer(
                transformers=[
                    ("num", numeric_transformer, numeric_features),
                    ("cat", categorical_transformer, categorical_features),
                ]
            ))]
        else:
            return []
//  [python]
def get_data_transformation(self):
        
        try:
            logging.info('Data Transformation initiated')
            
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']
            
            logging.info('Pipeline Initiated')
            
            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',Imputer(strategy='median')),
                ('scaler',StandardScaler())

                ]

            )
            
            # Categorigal Pipeline
            cat_pipeline=Pipeline(
                steps=[
                ('imputer',Imputer(strategy='most_frequent')),
                ('ordinalencoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]

            )
            
            preprocessor=ColumnTransformer([
            ('num_pipeline',num_pipeline,numerical_cols),
            ('cat_pipeline',cat_pipeline,categorical_cols)
            ])
            
            return preprocessor
            

            
            
        
        except Exception as e:
            logging.info("Exception occured in the initiate_datatransformation")

            raise customexception(e,sys)