jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _fit_standard_scaler(series, transform=False,
                             missing_numeric="drop",
                             imputation_strategy="mean"):
        imputer = None
        if series.isna().any():
            if missing_numeric == "drop":
                return None
            elif missing_numeric == "impute":
                imputer = Imputer(
                    missing_values=math.nan,
                    strategy=imputation_strategy)
        scaler = StandardScaler()
        if imputer is not None:
            encoder = Pipeline([('imputer', imputer), ('scaler', scaler)])
        else:
            encoder = scaler

        encoder.fit(
            np.reshape(series.to_list(), (series.shape[0], 1)))
        return encoder
//  [python]
def _fit_standard_scaler(series, transform=False,
                             missing_numeric="drop",
                             imputation_strategy="mean"):
        imputer = None
        if series.isna().any():
            if missing_numeric == "drop":
                return None
            elif missing_numeric == "impute":
                imputer = SimpleImputer(
                    missing_values=math.nan,
                    strategy=imputation_strategy)
        scaler = StandardScaler()
        if imputer is not None:
            encoder = Pipeline([('imputer', imputer), ('scaler', scaler)])
        else:
            encoder = scaler

        encoder.fit(
            np.reshape(series.to_list(), (series.shape[0], 1)))
        return encoder
//  [python]
def preprocess_support(data, prot='sex', fair_strategy=None):
  """Function to preprocess a loaded SUPPORT dataset.

  Imputation and standard rescaling is performed.

  Args:
    data:
      pandas dataframe of the loaded SUPPORT csv.
    prot:
      a string to idetify the name of the column containing the protected
      attributes.
    fair_strategy:
     Fairness strategy to be incorporated while preprocessing. Can be one of
     None, "unawaereness" or "coupled".

  Returns:
    a tuple of the shape of (x, t, e, a). Where 'x' are the features, t is the
    event (or censoring) times, e is the censoring indicator and a is a vector
    of the protected attributes.

  """

  x1 = data[[
      'age', 'num.co', 'meanbp', 'wblc', 'hrt', 'resp', 'temp', 'pafi', 'alb',
      'bili', 'crea', 'sod', 'ph', 'glucose', 'bun', 'urine', 'adlp', 'adls'
  ]]

  catfeats = ['sex', 'dzgroup', 'dzclass', 'income', 'race', 'ca']

  if fair_strategy in ['unawareness', 'coupled']:

    print('Note: For ' + fair_strategy, ' the protected feature is removed...')

    catfeats = set(catfeats)
    catfeats = list(catfeats - set([prot]))

  x2 = pd.get_dummies(data[catfeats])

  x = np.concatenate([x1, x2], axis=1)

  t = data['d.time'].values

  e = data['death'].values

  # Imputation

  x = Imputer(missing_values=np.nan, strategy='mean').fit_transform(x)

  # Standard scaling

  x = StandardScaler().fit_transform(x)

  a = data[prot].values

  remove = ~np.isnan(t)

  return x[remove], t[remove], e[remove], a[remove]
//  [python]
def preprocess_support(data, prot='sex', fair_strategy=None):
  """Function to preprocess a loaded SUPPORT dataset.

  Imputation and standard rescaling is performed.

  Args:
    data:
      pandas dataframe of the loaded SUPPORT csv.
    prot:
      a string to idetify the name of the column containing the protected
      attributes.
    fair_strategy:
     Fairness strategy to be incorporated while preprocessing. Can be one of
     None, "unawaereness" or "coupled".

  Returns:
    a tuple of the shape of (x, t, e, a). Where 'x' are the features, t is the
    event (or censoring) times, e is the censoring indicator and a is a vector
    of the protected attributes.

  """

  x1 = data[[
      'age', 'num.co', 'meanbp', 'wblc', 'hrt', 'resp', 'temp', 'pafi', 'alb',
      'bili', 'crea', 'sod', 'ph', 'glucose', 'bun', 'urine', 'adlp', 'adls'
  ]]

  catfeats = ['sex', 'dzgroup', 'dzclass', 'income', 'race', 'ca']

  if fair_strategy in ['unawareness', 'coupled']:

    print('Note: For ' + fair_strategy, ' the protected feature is removed...')

    catfeats = set(catfeats)
    catfeats = list(catfeats - set([prot]))

  x2 = pd.get_dummies(data[catfeats])

  x = np.concatenate([x1, x2], axis=1)

  t = data['d.time'].values

  e = data['death'].values

  # Imputation

  x = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(x)

  # Standard scaling

  x = StandardScaler().fit_transform(x)

  a = data[prot].values

  remove = ~np.isnan(t)

  return x[remove], t[remove], e[remove], a[remove]
//  [python]
def _modify_categorical_features(self, x):
        # prepare data for DT
        used_features = self.features
        if self.train_only_QI:
            used_features = self.quasi_identifiers
        numeric_features = [f for f in self.features if f in used_features and f not in self.categorical_features]
        categorical_features = [f for f in self.categorical_features if f in used_features]
        numeric_transformer = Pipeline(
            steps=[('imputer', Imputer(strategy='constant', fill_value=0))]
        )
        categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )
        encoded = preprocessor.fit_transform(x)
        return encoded
//  [python]
def _modify_categorical_features(self, x):
        # prepare data for DT
        used_features = self.features
        if self.train_only_QI:
            used_features = self.quasi_identifiers
        numeric_features = [f for f in self.features if f in used_features and f not in self.categorical_features]
        categorical_features = [f for f in self.categorical_features if f in used_features]
        numeric_transformer = Pipeline(
            steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
        )
        categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )
        encoded = preprocessor.fit_transform(x)
        return encoded
//  [python]
def preprocess_data(X_train, X_test, imputation_strategy='mean'):
    # Handling missing values
    if imputation_strategy == "constant":
        imputer = Imputer(strategy="constant", fill_value=0)
    else:
        imputer = Imputer(strategy=imputation_strategy)

    X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
    X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)

    # Separating categorical columns
    cat_cols = X_train.select_dtypes(include=['object']).columns.tolist()

    # Encoding categorical features
    if cat_cols:
        ohe = OneHotEncoder(drop='first', sparse=False)
        X_train_encoded = ohe.fit_transform(X_train[cat_cols])
        X_test_encoded = ohe.transform(X_test[cat_cols])

        # Replace categorical columns with their one-hot encoded values
        X_train = pd.concat([X_train.drop(cat_cols, axis=1), pd.DataFrame(X_train_encoded, columns=ohe.get_feature_names_out(cat_cols), index=X_train.index)], axis=1)
        X_test = pd.concat([X_test.drop(cat_cols, axis=1), pd.DataFrame(X_test_encoded, columns=ohe.get_feature_names_out(cat_cols), index=X_test.index)], axis=1)

    # Scaling numerical features
    scaler = MinMaxScaler()
    X_train = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns)
    X_test = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)

    return X_train, X_test
//  [python]
def preprocess_data(X_train, X_test, imputation_strategy='mean'):
    # Handling missing values
    if imputation_strategy == "constant":
        imputer = SimpleImputer(strategy="constant", fill_value=0)
    else:
        imputer = SimpleImputer(strategy=imputation_strategy)

    X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
    X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)

    # Separating categorical columns
    cat_cols = X_train.select_dtypes(include=['object']).columns.tolist()

    # Encoding categorical features
    if cat_cols:
        ohe = OneHotEncoder(drop='first', sparse=False)
        X_train_encoded = ohe.fit_transform(X_train[cat_cols])
        X_test_encoded = ohe.transform(X_test[cat_cols])

        # Replace categorical columns with their one-hot encoded values
        X_train = pd.concat([X_train.drop(cat_cols, axis=1), pd.DataFrame(X_train_encoded, columns=ohe.get_feature_names_out(cat_cols), index=X_train.index)], axis=1)
        X_test = pd.concat([X_test.drop(cat_cols, axis=1), pd.DataFrame(X_test_encoded, columns=ohe.get_feature_names_out(cat_cols), index=X_test.index)], axis=1)

    # Scaling numerical features
    scaler = MinMaxScaler()
    X_train = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns)
    X_test = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)

    return X_train, X_test
//  [python]
def get_pipeline(self, C: float, gamma: float) -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """

        model = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", Imputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", MinMaxScaler(feature_range=(0, 1)), ~self.categorical_data)])),
            ('svm',
             svm.SVC(gamma=gamma, C=C, random_state=self.rng, cache_size=self.cache_size))
        ])
        return model
//  [python]
def get_pipeline(self, C: float, gamma: float) -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """

        model = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", SimpleImputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", MinMaxScaler(feature_range=(0, 1)), ~self.categorical_data)])),
            ('svm',
             svm.SVC(gamma=gamma, C=C, random_state=self.rng, cache_size=self.cache_size))
        ])
        return model
//  [python]
def create_encoder(numeric_features, categorical_features, x):
    numeric_transformer = Pipeline(
        steps=[('imputer', Imputer(strategy='constant', fill_value=0))]
    )

    categorical_transformer = OneHotEncoder(handle_unknown="ignore")

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x)
    if scipy.sparse.issparse(encoded):
        pd.DataFrame.sparse.from_spmatrix(encoded)
    else:
        encoded = pd.DataFrame(encoded)

    return preprocessor, encoded
//  [python]
def create_encoder(numeric_features, categorical_features, x):
    numeric_transformer = Pipeline(
        steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
    )

    categorical_transformer = OneHotEncoder(handle_unknown="ignore")

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )
    encoded = preprocessor.fit_transform(x)
    if scipy.sparse.issparse(encoded):
        pd.DataFrame.sparse.from_spmatrix(encoded)
    else:
        encoded = pd.DataFrame(encoded)

    return preprocessor, encoded
//  [python]
def imputation(self, X, y, strategy):
        df = pd.concat([X, y], axis=1)
        columns = df.columns.tolist()
        df = pd.DataFrame(Imputer(missing_values=np.nan, strategy=strategy).fit_transform(df), columns=columns)
        try:
            target = y.name
        except:
            target = y.columns.tolist()[0]
        y = df[target]
        X = df[[col for col in df.columns if col != target]]
        return X, y
//  [python]
def imputation(self, X, y, strategy):
        df = pd.concat([X, y], axis=1)
        columns = df.columns.tolist()
        df = pd.DataFrame(SimpleImputer(missing_values=np.nan, strategy=strategy).fit_transform(df), columns=columns)
        try:
            target = y.name
        except:
            target = y.columns.tolist()[0]
        y = df[target]
        X = df[[col for col in df.columns if col != target]]
        return X, y
//  [python]
def preprocess(data, target, impute="simple", encode="label"):
    """Simple preprocessing pipeline for ML.

    Args:
        data: A Pandas dataframe
        target: Name of the target feature
        impute: Method to use for imputing numeric variables. Only 'simple' is implemented.
        encode: Method to use for encoding categorical variables. Only 'label' is implemented.

    Raises:
        NotImplementedError: Imputation or encoding method not implemented.
        ValueError: No columns left to preprocess.

    Returns:
        (X, y) tuple of numpy arrays
    """
    y = data[target]
    data = data.drop(target, axis=1)

    data = data.dropna(axis=1, how="all")

    # Process numeric features
    num = data.select_dtypes(["number"])
    if num.shape[1] > 0:
        if impute == "simple":
            imp = Imputer(missing_values=np.nan, strategy="mean")
            x_num = imp.fit_transform(num)
        else:
            raise NotImplementedError("Unknown imputation method: {}".format(impute))

    # Ordinal encode everything else
    # TODO: Address date and text columns
    cat = data[[c for c in data.columns if c not in num.columns]]
    if cat.shape[1] > 0:
        cat = cat.astype(str)
        cat = cat.fillna("")
        if encode == "label":
            x_cat = cat.apply(LabelEncoder().fit_transform)
        else:
            raise NotImplementedError("Unknown encoding method: {}".format(encode))

    if num.shape[1] > 0 and cat.shape[1] > 0:
        X = pd.DataFrame(
            np.concatenate([x_num, x_cat], axis=1),
            columns=list(num.columns.values) + list(cat.columns),
        )
    elif num.shape[1] > 0:
        X = pd.DataFrame(x_num, columns=num.columns)
    elif cat.shape[1] > 0:
        X = pd.DataFrame(x_cat, columns=x_cat.columns)
    else:
        raise ValueError("No numeric or categorical columns were found.")

    return X, y
//  [python]
def preprocess(data, target, impute="simple", encode="label"):
    """Simple preprocessing pipeline for ML.

    Args:
        data: A Pandas dataframe
        target: Name of the target feature
        impute: Method to use for imputing numeric variables. Only 'simple' is implemented.
        encode: Method to use for encoding categorical variables. Only 'label' is implemented.

    Raises:
        NotImplementedError: Imputation or encoding method not implemented.
        ValueError: No columns left to preprocess.

    Returns:
        (X, y) tuple of numpy arrays
    """
    y = data[target]
    data = data.drop(target, axis=1)

    data = data.dropna(axis=1, how="all")

    # Process numeric features
    num = data.select_dtypes(["number"])
    if num.shape[1] > 0:
        if impute == "simple":
            imp = SimpleImputer(missing_values=np.nan, strategy="mean")
            x_num = imp.fit_transform(num)
        else:
            raise NotImplementedError("Unknown imputation method: {}".format(impute))

    # Ordinal encode everything else
    # TODO: Address date and text columns
    cat = data[[c for c in data.columns if c not in num.columns]]
    if cat.shape[1] > 0:
        cat = cat.astype(str)
        cat = cat.fillna("")
        if encode == "label":
            x_cat = cat.apply(LabelEncoder().fit_transform)
        else:
            raise NotImplementedError("Unknown encoding method: {}".format(encode))

    if num.shape[1] > 0 and cat.shape[1] > 0:
        X = pd.DataFrame(
            np.concatenate([x_num, x_cat], axis=1),
            columns=list(num.columns.values) + list(cat.columns),
        )
    elif num.shape[1] > 0:
        X = pd.DataFrame(x_num, columns=num.columns)
    elif cat.shape[1] > 0:
        X = pd.DataFrame(x_cat, columns=x_cat.columns)
    else:
        raise ValueError("No numeric or categorical columns were found.")

    return X, y
//  [python]
def _build_model():
    """Build the model to fit."""
    numeric_transformer = Imputer()
    categorical_transformer = Pipeline(
        steps=[('imputer', Imputer(strategy='most_frequent')), ('encoder', OrdinalEncoder())]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, _NUM_FEATURES),
            ('cat', categorical_transformer, _CAT_FEATURES),
        ]
    )

    model = Pipeline(
        steps=[
            ('preprocessing', preprocessor),
            ('model', RandomForestClassifier(max_depth=5, n_jobs=-1, random_state=0))
        ]
    )

    return model
//  [python]
def _build_model():
    """Build the model to fit."""
    numeric_transformer = SimpleImputer()
    categorical_transformer = Pipeline(
        steps=[('imputer', SimpleImputer(strategy='most_frequent')), ('encoder', OrdinalEncoder())]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, _NUM_FEATURES),
            ('cat', categorical_transformer, _CAT_FEATURES),
        ]
    )

    model = Pipeline(
        steps=[
            ('preprocessing', preprocessor),
            ('model', RandomForestClassifier(max_depth=5, n_jobs=-1, random_state=0))
        ]
    )

    return model
//  [python]
def _preprocessed_data(self):
        num_pipeline = make_pipeline(Imputer(strategy='constant', fill_value=-9999))
        cat_pipeline = make_pipeline(Imputer(strategy='most_frequent'),
                                     OneHotEncoder(handle_unknown='ignore', sparse=False))
        full_pipe = ColumnTransformer(transformers=[
            ('num_pipe', num_pipeline, make_column_selector(dtype_include=np.number)),
            ('cat_pipe', cat_pipeline, make_column_selector(dtype_exclude=np.number))
        ])
        X = full_pipe.fit_transform(self.data.X)

        return X, self.data.y
//  [python]
def _preprocessed_data(self):
        num_pipeline = make_pipeline(SimpleImputer(strategy='constant', fill_value=-9999))
        cat_pipeline = make_pipeline(SimpleImputer(strategy='most_frequent'),
                                     OneHotEncoder(handle_unknown='ignore', sparse=False))
        full_pipe = ColumnTransformer(transformers=[
            ('num_pipe', num_pipeline, make_column_selector(dtype_include=np.number)),
            ('cat_pipe', cat_pipeline, make_column_selector(dtype_exclude=np.number))
        ])
        X = full_pipe.fit_transform(self.data.X)

        return X, self.data.y
//  [python]
def impute_data(*inputs, strategy='median'):
    """
    Imputes missing data in ``inputs``

    Parameters
    ----------
    inputs : list of pandas.DataFrame
        List of input data
    strategy : {'mean', 'median'}, optional
        Imputation strategy. Default: 'median'

    Returns
    -------
    ouputs : list of pandas.DataFrame
        List of imputed data
    """

    from sklearn.preprocessing import Imputer

    impute = Imputer(strategy=strategy)
    outputs = [pd.DataFrame(impute.fit_transform(f), columns=f.columns,
                            index=f.index) for f in inputs]

    return outputs
//  [python]
def impute_data(*inputs, strategy='median'):
    """
    Imputes missing data in ``inputs``

    Parameters
    ----------
    inputs : list of pandas.DataFrame
        List of input data
    strategy : {'mean', 'median'}, optional
        Imputation strategy. Default: 'median'

    Returns
    -------
    ouputs : list of pandas.DataFrame
        List of imputed data
    """

    from sklearn.impute import SimpleImputer

    # https://github.com/scikit-learn/scikit-learn/pull/9212
    impute = SimpleImputer(strategy=strategy)
    outputs = [pd.DataFrame(impute.fit_transform(f), columns=f.columns,
                            index=f.index) for f in inputs]

    return outputs
//  [python]
def __init__(self, random_state: int = 0) -> None:
        super().__init__(random_state=random_state)

        self._model = Imputer(strategy="most_frequent")
//  [python]
def __init__(self, random_state: int = 0) -> None:
        super().__init__(random_state=random_state)

        self._model = SimpleImputer(strategy="most_frequent")
//  [python]
def get_data_transformer_object(self):
        
        '''
        This function is transforming the data
        '''
        
        try:
            numerical_columns = configure["data_transformation"]["numerical"]["columns"]
            categorical_columns = configure["data_transformation"]["categorical"]["columns"]
            
            numerical_pipeline = Pipeline(
                steps=[
                    ("imputer",Imputer(strategy="mean")),
                    ('scaler',StandardScaler(with_mean=False))
                ]
            )
            
            categorical_pipeline = Pipeline(
                steps=[
                    ('imputer',Imputer(strategy='most_frequent')),
                    ('one_hot_encoder', OneHotEncoder()),
                    ('scaler',StandardScaler(with_mean=False))
                ]
            )
            
            logging.info('Numerical columns scaling completed!')
            logging.info("Categorical columns encoding completed!")
            
            preprocessor = ColumnTransformer(
                [
                    ("num_pipeline",numerical_pipeline,numerical_columns),
                    ("cat_pipeline",categorical_pipeline,categorical_columns)
                ]
            )
            
            return preprocessor
        
        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_data_transformer_object(self):
        
        '''
        This function is transforming the data
        '''
        
        try:
            numerical_columns = configure["data_transformation"]["numerical"]["columns"]
            categorical_columns = configure["data_transformation"]["categorical"]["columns"]
            
            numerical_pipeline = Pipeline(
                steps=[
                    ("imputer",SimpleImputer(strategy="mean")),
                    ('scaler',StandardScaler(with_mean=False))
                ]
            )
            
            categorical_pipeline = Pipeline(
                steps=[
                    ('imputer',SimpleImputer(strategy='most_frequent')),
                    ('one_hot_encoder', OneHotEncoder()),
                    ('scaler',StandardScaler(with_mean=False))
                ]
            )
            
            logging.info('Numerical columns scaling completed!')
            logging.info("Categorical columns encoding completed!")
            
            preprocessor = ColumnTransformer(
                [
                    ("num_pipeline",numerical_pipeline,numerical_columns),
                    ("cat_pipeline",categorical_pipeline,categorical_columns)
                ]
            )
            
            return preprocessor
        
        except Exception as e:
            raise CustomException(e,sys)
//  [python]
def get_tabular_preprocessors() -> Dict[str, List[BaseEstimator]]:
    """
    This function creates a Dictionary containing a list
    of numerical and categorical preprocessors

    Returns:
        Dict[str, List[BaseEstimator]]
    """
    preprocessors: Dict[str, List[BaseEstimator]] = dict()

    # Categorical Preprocessors
    onehot_encoder = preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value',
                                                  unknown_value=-1)
    categorical_imputer = Imputer(strategy='constant', copy=False)

    preprocessors['categorical'] = [categorical_imputer, onehot_encoder]

    return preprocessors
//  [python]
def get_tabular_preprocessors() -> Dict[str, List[BaseEstimator]]:
    """
    This function creates a Dictionary containing a list
    of numerical and categorical preprocessors

    Returns:
        Dict[str, List[BaseEstimator]]
    """
    preprocessors: Dict[str, List[BaseEstimator]] = dict()

    # Categorical Preprocessors
    onehot_encoder = preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value',
                                                  unknown_value=-1)
    categorical_imputer = SimpleImputer(strategy='constant', copy=False)

    preprocessors['categorical'] = [categorical_imputer, onehot_encoder]

    return preprocessors
//  [python]
def enron_raw_model(enron_raw_data: pd.DataFrame) -> Tuple[Pipeline, List[str]]:
    timer = Timer()

    columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"]

    numeric_transformer = Pipeline([("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())])

    columns_to_encode = [key for key in input_types.keys() if input_types[key] == "category"]

    categorical_transformer = Pipeline(
        [
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    text_transformer = Pipeline([("vect", CountVectorizer()), ("tfidf", TfidfTransformer())])

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, columns_to_scale),
            ("cat", categorical_transformer, columns_to_encode),
            ("text_Mail", text_transformer, "Content"),
        ]
    )
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=100, random_state=30))]
    )

    Y = enron_raw_data["Target"]
    X = enron_raw_data.drop(columns="Target")
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.20, random_state=30, stratify=Y  # NOSONAR
    )
    clf.fit(X_train, Y_train)

    model_score = clf.score(X_test, Y_test)
    timer.stop(f"Trained model with score: {model_score}")
    return clf, list(input_types.keys())
//  [python]
def enron_raw_model(enron_raw_data: pd.DataFrame) -> Tuple[Pipeline, List[str]]:
    timer = Timer()

    columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"]

    numeric_transformer = Pipeline([("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())])

    columns_to_encode = [key for key in input_types.keys() if input_types[key] == "category"]

    categorical_transformer = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    text_transformer = Pipeline([("vect", CountVectorizer()), ("tfidf", TfidfTransformer())])

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, columns_to_scale),
            ("cat", categorical_transformer, columns_to_encode),
            ("text_Mail", text_transformer, "Content"),
        ]
    )
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=100, random_state=30))]
    )

    Y = enron_raw_data["Target"]
    X = enron_raw_data.drop(columns="Target")
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.20, random_state=30, stratify=Y  # NOSONAR
    )
    clf.fit(X_train, Y_train)

    model_score = clf.score(X_test, Y_test)
    timer.stop(f"Trained model with score: {model_score}")
    return clf, list(input_types.keys())
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with SimpleImputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline(
        [
            ("imputer", Imputer(strategy="mean", missing_values=np.nan)),
            ("classifier", MockClassifier()),
        ]
    )
    GridSearchCV(p, {"classifier__foo_param": [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with SimpleImputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean", missing_values=np.nan)),
            ("classifier", MockClassifier()),
        ]
    )
    GridSearchCV(p, {"classifier__foo_param": [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
plt.show()

X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  Imputer(strategy="median"), StandardScaler())
//  [python]
plt.show()

X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def pre_processing_pipeline(input_files, out_file): 
    
    try:
        os.remove(out_file)
    except OSError:
        pass
    
    for ix, my_file in enumerate(input_files):
        my_dataset = pd.read_csv(my_file, sep=';')
        
        try:
            my_dataset['FORCE_2020_LITHOFACIES_LITHOLOGY'].to_hdf(
                out_file, key=my_file[:-4] + '_target') 
        except:
            my_target = pd.read_csv('leaderboard_test_target.csv', sep=';')
            my_target['FORCE_2020_LITHOFACIES_LITHOLOGY'].to_hdf(
                out_file, key=my_file[:-4] + '_target') 
                    
        if ix==0:    
            # Fitting the categorical encoders
            my_encoder = OrdinalEncoder()
            my_encoder.set_params(handle_unknown='use_encoded_value', 
                                  unknown_value=-1, 
                                  encoded_missing_value=-1).fit(
                                      my_dataset[['FORMATION']])

        my_dataset = (my_dataset.
                        pipe(replace_inf).
                        pipe(log_transform).
                        pipe(calculate_delta).
                        pipe(categorical_encoder, 
                             my_encoder=my_encoder, cols=['FORMATION']).
                        pipe(feature_selection))
        my_dataset.to_hdf(out_file, key=my_file[:-4])  
        
        imputer = Imputer(missing_values=np.nan, strategy='mean')
        imputer = imputer.fit(my_dataset[my_dataset.columns])
        my_dataset[my_dataset.columns] = imputer.transform(
            my_dataset[my_dataset.columns])
        my_dataset.to_hdf(out_file, key= my_file[:-4])
//  [python]
def pre_processing_pipeline(input_files, out_file): 
    
    try:
        os.remove(out_file)
    except OSError:
        pass
    
    for ix, my_file in enumerate(input_files):
        my_dataset = pd.read_csv(my_file, sep=';')
        
        try:
            my_dataset['FORCE_2020_LITHOFACIES_LITHOLOGY'].to_hdf(
                out_file, key=my_file[:-4] + '_target') 
        except:
            my_target = pd.read_csv('leaderboard_test_target.csv', sep=';')
            my_target['FORCE_2020_LITHOFACIES_LITHOLOGY'].to_hdf(
                out_file, key=my_file[:-4] + '_target') 
                    
        if ix==0:    
            # Fitting the categorical encoders
            my_encoder = OrdinalEncoder()
            my_encoder.set_params(handle_unknown='use_encoded_value', 
                                  unknown_value=-1, 
                                  encoded_missing_value=-1).fit(
                                      my_dataset[['FORMATION']])

        my_dataset = (my_dataset.
                        pipe(replace_inf).
                        pipe(log_transform).
                        pipe(calculate_delta).
                        pipe(categorical_encoder, 
                             my_encoder=my_encoder, cols=['FORMATION']).
                        pipe(feature_selection))
        my_dataset.to_hdf(out_file, key=my_file[:-4])  
        
        imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
        imputer = imputer.fit(my_dataset[my_dataset.columns])
        my_dataset[my_dataset.columns] = imputer.transform(
            my_dataset[my_dataset.columns])
        my_dataset.to_hdf(out_file, key= my_file[:-4])
//  [python]
def fit(self, df):
        self.transformer = Imputer(strategy="mean")
        train_data = df.query("is_train")[["Age"]]
        self.transformer.fit(train_data)
//  [python]
def fit(self, df):
        self.transformer = SimpleImputer(strategy="mean")
        train_data = df.query("is_train")[["Age"]]
        self.transformer.fit(train_data)
//  [python]
def clustered_map(year, n_clusters, indicators):
    if not indicators:
        raise PreventUpdate
    imp = Imputer(missing_values=np.nan, strategy='mean')
    scaler = StandardScaler()
    kmeans = KMeans(n_clusters=n_clusters)
    
    df = poverty[poverty['is_country'] & poverty['year'].eq(year)][indicators + ['Country Name', 'year']]
    data = df[indicators]
    if df.isna().all().any():
        return px.scatter(title='No available data for the selected combination of year/indicators.')
    data_no_na = imp.fit_transform(data)
    scaled_data = scaler.fit_transform(data_no_na)
    kmeans.fit(scaled_data)

    fig = px.choropleth(df,
                        locations='Country Name',
                        locationmode='country names',
                        color=[str(x) for x in  kmeans.labels_], 
                        labels={'color': 'Cluster'},
                        hover_data=indicators,
                        height=650,
                        title=f'Country clusters - {year}. Number of clusters: {n_clusters}<br>Inertia: {kmeans.inertia_:,.2f}',
                        color_discrete_sequence=px.colors.qualitative.T10)
    fig.add_annotation(x=-0.1, y=-0.15, 
                       xref='paper', yref='paper',
                       text='Indicators:<br>' + "<br>".join(indicators), 
                       showarrow=False)
    fig.layout.geo.showframe = False
    fig.layout.geo.showcountries = True
    fig.layout.geo.projection.type = 'natural earth'
    fig.layout.geo.lataxis.range = [-53, 76]
    fig.layout.geo.lonaxis.range = [-137, 168]
    fig.layout.geo.landcolor = 'white'
    fig.layout.geo.bgcolor = '#E5ECF6'
    fig.layout.paper_bgcolor = '#E5ECF6'
    fig.layout.geo.countrycolor = 'gray'
    fig.layout.geo.coastlinecolor = 'gray'
    return fig
//  [python]
def clustered_map(year, n_clusters, indicators):
    if not indicators:
        raise PreventUpdate
    imp = SimpleImputer(missing_values=np.nan, strategy='mean')
    scaler = StandardScaler()
    kmeans = KMeans(n_clusters=n_clusters)
    
    df = poverty[poverty['is_country'] & poverty['year'].eq(year)][indicators + ['Country Name', 'year']]
    data = df[indicators]
    if df.isna().all().any():
        return px.scatter(title='No available data for the selected combination of year/indicators.')
    data_no_na = imp.fit_transform(data)
    scaled_data = scaler.fit_transform(data_no_na)
    kmeans.fit(scaled_data)

    fig = px.choropleth(df,
                        locations='Country Name',
                        locationmode='country names',
                        color=[str(x) for x in  kmeans.labels_], 
                        labels={'color': 'Cluster'},
                        hover_data=indicators,
                        height=650,
                        title=f'Country clusters - {year}. Number of clusters: {n_clusters}<br>Inertia: {kmeans.inertia_:,.2f}',
                        color_discrete_sequence=px.colors.qualitative.T10)
    fig.add_annotation(x=-0.1, y=-0.15, 
                       xref='paper', yref='paper',
                       text='Indicators:<br>' + "<br>".join(indicators), 
                       showarrow=False)
    fig.layout.geo.showframe = False
    fig.layout.geo.showcountries = True
    fig.layout.geo.projection.type = 'natural earth'
    fig.layout.geo.lataxis.range = [-53, 76]
    fig.layout.geo.lonaxis.range = [-137, 168]
    fig.layout.geo.landcolor = 'white'
    fig.layout.geo.bgcolor = '#E5ECF6'
    fig.layout.paper_bgcolor = '#E5ECF6'
    fig.layout.geo.countrycolor = 'gray'
    fig.layout.geo.coastlinecolor = 'gray'
    return fig
//  [python]
def __call__(self, data):
        from Orange.data.sql.table import SqlTable
        if isinstance(data, SqlTable):
            return Impute()(data)
        imputer = sklearn.preprocessing.Imputer(strategy=self.strategy)
        imputer.fit(data.X)
        # Create new variables with appropriate `compute_value`, but
        # drop the ones which do not have valid `imputer.statistics_`
        # (i.e. all NaN columns). `sklearn.preprocessing.Imputer` already
        # drops them from the transformed X.
        features = [var.copy(compute_value=impute.ReplaceUnknowns(var, value))
                    for var, value in zip(data.domain.attributes,
                                          imputer.statistics_)
                    if not np.isnan(value)]
        domain = Orange.data.Domain(features, data.domain.class_vars,
                                    data.domain.metas)
        new_data = data.transform(domain)
        return new_data
//  [python]
def __call__(self, data):
        from Orange.data.sql.table import SqlTable
        if isinstance(data, SqlTable):
            return Impute()(data)
        imputer = SimpleImputer(strategy=self.strategy)
        imputer.fit(data.X)
        # Create new variables with appropriate `compute_value`, but
        # drop the ones which do not have valid `imputer.statistics_`
        # (i.e. all NaN columns). `sklearn.preprocessing.Imputer` already
        # drops them from the transformed X.
        features = [var.copy(compute_value=impute.ReplaceUnknowns(var, value))
                    for var, value in zip(data.domain.attributes,
                                          imputer.statistics_)
                    if not np.isnan(value)]
        domain = Orange.data.Domain(features, data.domain.class_vars,
                                    data.domain.metas)
        new_data = data.transform(domain)
        return new_data
//  [python]
def test_imputation_deletion_warning(strategy):
    X = np.ones((3, 5))
    X[:, 0] = np.nan
    imputer = Imputer(strategy=strategy, verbose=1)

    # TODO: Remove in 1.3
    with pytest.warns(FutureWarning, match="The 'verbose' parameter"):
        imputer.fit(X)

    with pytest.warns(UserWarning, match="Skipping"):
        imputer.transform(X)
//  [python]
def test_imputation_deletion_warning(strategy):
    X = np.ones((3, 5))
    X[:, 0] = np.nan
    imputer = SimpleImputer(strategy=strategy, verbose=1)

    # TODO: Remove in 1.3
    with pytest.warns(FutureWarning, match="The 'verbose' parameter"):
        imputer.fit(X)

    with pytest.warns(UserWarning, match="Skipping"):
        imputer.transform(X)
//  [python]
print(f"full : {sum(y_heart)} CHD / {len(y_heart)} samples")
print(f"train: {sum(y_heart_train)} CHD / {len(y_heart_train)} samples")
print(f"test : {sum(y_heart_test)} CHD / {len(y_heart_test)} samples")
print("\n***\n")

## creating the pipeline and grid, and fitting it
# don't forget the imputer and the scaler 
pipeline_heart=Pipeline([('imputer',Imputer(strategy='mean')),
                            ('scalar',StandardScaler()),
                            ('model',LogisticRegression())])
//  [python]
print(f"full : {sum(y_heart)} CHD / {len(y_heart)} samples")
print(f"train: {sum(y_heart_train)} CHD / {len(y_heart_train)} samples")
print(f"test : {sum(y_heart_test)} CHD / {len(y_heart_test)} samples")
print("\n***\n")

## creating the pipeline and grid, and fitting it
# don't forget the imputer and the scaler 
pipeline_heart=Pipeline([('imputer',SimpleImputer(strategy='mean')),
                            ('scalar',StandardScaler()),
                            ('model',LogisticRegression())])
//  [python]
def test_imputation_deletion_warning_feature_names(strategy):

    pd = pytest.importorskip("pandas")

    missing_values = np.nan
    feature_names = np.array(["a", "b", "c", "d"], dtype=object)
    X = pd.DataFrame(
        [
            [missing_values, missing_values, 1, missing_values],
            [4, missing_values, 2, 10],
        ],
        columns=feature_names,
    )

    imputer = Imputer(strategy=strategy, verbose=1)

    # TODO: Remove in 1.3
    with pytest.warns(FutureWarning, match="The 'verbose' parameter"):
        imputer.fit(X)

    # check SimpleImputer returning feature name attribute correctly
    assert_array_equal(imputer.feature_names_in_, feature_names)

    # ensure that skipped feature warning includes feature name
    with pytest.warns(
        UserWarning, match=r"Skipping features without any observed values: \['b'\]"
    ):
        imputer.transform(X)
//  [python]
def test_imputation_deletion_warning_feature_names(strategy):

    pd = pytest.importorskip("pandas")

    missing_values = np.nan
    feature_names = np.array(["a", "b", "c", "d"], dtype=object)
    X = pd.DataFrame(
        [
            [missing_values, missing_values, 1, missing_values],
            [4, missing_values, 2, 10],
        ],
        columns=feature_names,
    )

    imputer = SimpleImputer(strategy=strategy, verbose=1)

    # TODO: Remove in 1.3
    with pytest.warns(FutureWarning, match="The 'verbose' parameter"):
        imputer.fit(X)

    # check SimpleImputer returning feature name attribute correctly
    assert_array_equal(imputer.feature_names_in_, feature_names)

    # ensure that skipped feature warning includes feature name
    with pytest.warns(
        UserWarning, match=r"Skipping features without any observed values: \['b'\]"
    ):
        imputer.transform(X)
//  [python]
def test_imputer_lists_fit_transform():
    """Check transform uses object dtype when fitted on an object dtype.

    Non-regression test for #19572.
    """

    X = [["a", "b"], ["c", "b"], ["a", "a"]]
    imp_frequent = Imputer(strategy="most_frequent").fit(X)
    X_trans = imp_frequent.transform([[np.nan, np.nan]])
    assert X_trans.dtype == object
    assert_array_equal(X_trans, [["a", "b"]])
//  [python]
def test_imputer_lists_fit_transform():
    """Check transform uses object dtype when fitted on an object dtype.

    Non-regression test for #19572.
    """

    X = [["a", "b"], ["c", "b"], ["a", "a"]]
    imp_frequent = SimpleImputer(strategy="most_frequent").fit(X)
    X_trans = imp_frequent.transform([[np.nan, np.nan]])
    assert X_trans.dtype == object
    assert_array_equal(X_trans, [["a", "b"]])
//  [python]
def test_imputer_transform_preserves_numeric_dtype(dtype_test):
    """Check transform preserves numeric dtype independent of fit dtype."""
    X = np.asarray(
        [[1.2, 3.4, np.nan], [np.nan, 1.2, 1.3], [4.2, 2, 1]], dtype=np.float64
    )
    imp = Imputer().fit(X)

    X_test = np.asarray([[np.nan, np.nan, np.nan]], dtype=dtype_test)
    X_trans = imp.transform(X_test)
    assert X_trans.dtype == dtype_test
//  [python]
def test_imputer_transform_preserves_numeric_dtype(dtype_test):
    """Check transform preserves numeric dtype independent of fit dtype."""
    X = np.asarray(
        [[1.2, 3.4, np.nan], [np.nan, 1.2, 1.3], [4.2, 2, 1]], dtype=np.float64
    )
    imp = SimpleImputer().fit(X)

    X_test = np.asarray([[np.nan, np.nan, np.nan]], dtype=dtype_test)
    X_trans = imp.transform(X_test)
    assert X_trans.dtype == dtype_test
//  [python]
def fill_na_num(self):
        return Imputer(strategy="constant", 
                             fill_value=self.fill_num_na_value)
//  [python]
def fill_na_num(self):
        return SimpleImputer(strategy="constant", 
                             fill_value=self.fill_num_na_value)
//  [python]
def test_simple_imputer_keep_empty_features(strategy, array_type, keep_empty_features):
    """Check the behaviour of `keep_empty_features` with all strategies but
    'constant'.
    """
    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
    X = _convert_container(X, array_type)
    imputer = Imputer(strategy=strategy, keep_empty_features=keep_empty_features)

    for method in ["fit_transform", "transform"]:
        X_imputed = getattr(imputer, method)(X)
        if keep_empty_features:
            assert X_imputed.shape == X.shape
            constant_feature = (
                X_imputed[:, 0].A if array_type == "sparse" else X_imputed[:, 0]
            )
            assert_array_equal(constant_feature, 0)
        else:
            assert X_imputed.shape == (X.shape[0], X.shape[1] - 1)
//  [python]
def test_simple_imputer_keep_empty_features(strategy, array_type, keep_empty_features):
    """Check the behaviour of `keep_empty_features` with all strategies but
    'constant'.
    """
    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
    X = _convert_container(X, array_type)
    imputer = SimpleImputer(strategy=strategy, keep_empty_features=keep_empty_features)

    for method in ["fit_transform", "transform"]:
        X_imputed = getattr(imputer, method)(X)
        if keep_empty_features:
            assert X_imputed.shape == X.shape
            constant_feature = (
                X_imputed[:, 0].A if array_type == "sparse" else X_imputed[:, 0]
            )
            assert_array_equal(constant_feature, 0)
        else:
            assert X_imputed.shape == (X.shape[0], X.shape[1] - 1)
//  [python]
def test_simple_imputer_constant_keep_empty_features(array_type, keep_empty_features):
    """Check the behaviour of `keep_empty_features` with `strategy='constant'.
    For backward compatibility, a column full of missing values will always be
    fill and never dropped.
    """
    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
    X = _convert_container(X, array_type)
    fill_value = 10
    imputer = Imputer(
        strategy="constant",
        fill_value=fill_value,
        keep_empty_features=keep_empty_features,
    )

    for method in ["fit_transform", "transform"]:
        X_imputed = getattr(imputer, method)(X)
        assert X_imputed.shape == X.shape
        constant_feature = (
            X_imputed[:, 0].A if array_type == "sparse" else X_imputed[:, 0]
        )
        assert_array_equal(constant_feature, fill_value)
//  [python]
def test_simple_imputer_constant_keep_empty_features(array_type, keep_empty_features):
    """Check the behaviour of `keep_empty_features` with `strategy='constant'.
    For backward compatibility, a column full of missing values will always be
    fill and never dropped.
    """
    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
    X = _convert_container(X, array_type)
    fill_value = 10
    imputer = SimpleImputer(
        strategy="constant",
        fill_value=fill_value,
        keep_empty_features=keep_empty_features,
    )

    for method in ["fit_transform", "transform"]:
        X_imputed = getattr(imputer, method)(X)
        assert X_imputed.shape == X.shape
        constant_feature = (
            X_imputed[:, 0].A if array_type == "sparse" else X_imputed[:, 0]
        )
        assert_array_equal(constant_feature, fill_value)
//  [python]
def fill_na_cat(self):
        pipe = Pipeline(
                steps=[("impute", Imputer(strategy=self.fill_cat_na_strategy)),
                       ("onehot", OneHotEncoder(handle_unknown="ignore")),
                       ])
        return pipe
//  [python]
def fill_na_cat(self):
        pipe = Pipeline(
                steps=[("impute", SimpleImputer(strategy=self.fill_cat_na_strategy)),
                       ("onehot", OneHotEncoder(handle_unknown="ignore")),
                       ])
        return pipe