jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def fit(self, x, y=None):
        """Fit the imputer on the input data."""
        _ = check_array(x, force_all_finite="allow-nan")
        if not isinstance(x, pd.DataFrame):
            x = pd.DataFrame(np.asarray(x))

        self.in_feature_names = list(x.columns)
        self._n_in_features = x.shape[1]

        # Remove always null columns
        is_column_null = x.isnull().all(axis="index")
        self.non_null_feature_names = list(x.columns[~is_column_null])

        # Build the proper imputation transformer
        # - Identity function if strategy is None
        # - SimpleImputer with the dedicated strategy
        if self.imputation_strategy is None:
            self.imputer_ = FunctionTransformer(func=self._identity)
        else:
            self.imputer_ = Imputer(
                missing_values=self.missing_values,
                strategy=self.imputation_strategy,
                fill_value=self.fill_value,
            ).set_output(transform="pandas")

        # Imputers do not support labels
        self.imputer_.fit(X=x, y=None)
//  [python]
def fit(self, x, y=None):
        """Fit the imputer on the input data."""
        _ = check_array(x, force_all_finite="allow-nan")
        if not isinstance(x, pd.DataFrame):
            x = pd.DataFrame(np.asarray(x))

        self.in_feature_names = list(x.columns)
        self._n_in_features = x.shape[1]

        # Remove always null columns
        is_column_null = x.isnull().all(axis="index")
        self.non_null_feature_names = list(x.columns[~is_column_null])

        # Build the proper imputation transformer
        # - Identity function if strategy is None
        # - SimpleImputer with the dedicated strategy
        if self.imputation_strategy is None:
            self.imputer_ = FunctionTransformer(func=self._identity)
        else:
            self.imputer_ = SimpleImputer(
                missing_values=self.missing_values,
                strategy=self.imputation_strategy,
                fill_value=self.fill_value,
            ).set_output(transform="pandas")

        # Imputers do not support labels
        self.imputer_.fit(X=x, y=None)
//  [python]
def make_tabpfn(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), Imputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False,
                                         max_categories=10), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, TabPFNClassifier())
//  [python]
def make_tabpfn(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), SimpleImputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False,
                                         max_categories=10), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, TabPFNClassifier())
//  [python]
def make_mlp(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), Imputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, TorchMLP(n_epochs=100))
//  [python]
def make_mlp(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), SimpleImputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, TorchMLP(n_epochs=100))
//  [python]
def make_distilled_tabpfn(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), Imputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False,
                                         max_categories=10), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, DistilledTabPFNMLP(n_epochs=100))
//  [python]
def make_distilled_tabpfn(categorical_features):
    cont_pipe = make_pipeline(StandardScaler(), SimpleImputer())
    preprocess = make_column_transformer((OneHotEncoder(handle_unknown='ignore', sparse_output=False,
                                         max_categories=10), categorical_features), remainder=cont_pipe)
    return make_pipeline(preprocess, DistilledTabPFNMLP(n_epochs=100))
//  [python]
def fit(self,
            X: PIPELINE_DATA_DTYPE,
            y: Optional[PIPELINE_DATA_DTYPE] = None) -> 'NumericalImputation':
        import sklearn.preprocessing

        self.preprocessor = sklearn.preprocessing.Imputer(
            strategy=self.strategy, copy=False)
        self.preprocessor.fit(X)
        return self
//  [python]
def fit(self,
            X: PIPELINE_DATA_DTYPE,
            y: Optional[PIPELINE_DATA_DTYPE] = None) -> 'NumericalImputation':
        import sklearn.impute

        self.preprocessor = sklearn.impute.SimpleImputer(
            strategy=self.strategy, copy=False)
        self.preprocessor.fit(X)
        return self
//  [python]
def pre_process(raw_data, categorical, impute=True, standardize=True, one_hot_encode=True):
    """
    Pre-process one dataset.

    Args:
        raw_data (np.ndarray):    raw features of the n-by-d dataset, without indices and headings.
        categorical (list):       a boolean list of length d indicating whether each raw feature is categorical.
        impute (bool):            whether to impute missing entries or not.
        standardize (bool):       whether to standardize each feature or not.
        one_hot_encode (bool):    whether to use one hot encoding to pre-process categorical features or not.
    Returns:
        np.ndarray:               pre-processed dataset.
    """
    # list of pre-processed arrays (sub-portions of dataset)
    processed = []

    # whether to impute missing entries
    if impute:
        # if there are any categorical features
        if np.array(categorical).any():
            raw_categorical = raw_data[:, categorical]
            # impute missing entries in categorical features using the most frequent number
            imp_categorical = Imputer(strategy='most_frequent', copy=False)
            processed.append(imp_categorical.fit_transform(raw_categorical))

        # if there are any numeric features
        if np.invert(categorical).any():
            raw_numeric = raw_data[:, np.invert(categorical)]
            # impute missing entries in non-categorical features using mean
            imp_numeric = Imputer(strategy='mean', copy=False)
            processed.append(imp_numeric.fit_transform(raw_numeric))

        # data has now been re-ordered so all categorical features appear first
        categorical = np.array(sorted(categorical, reverse=True))
        processed_data = np.hstack(tuple(processed))

    else:
        processed_data = raw_data

    # one-hot encoding for categorical features (only if there exist any)
    if one_hot_encode and np.array(categorical).any():
        categorical_columns = list(np.where(categorical)[0])
#         encoder = OneHotEncoder(categorical_features=categorical)
        encoder = ColumnTransformer([('one_hot_encoder', OneHotEncoder(), categorical_columns)], 
                        remainder='passthrough', sparse_threshold=0)
        processed_data = encoder.fit_transform(processed_data)
        categorical = np.zeros(processed_data.shape[1], dtype=bool)
    
    # standardize all numeric and one-hot encoded categorical features
    if standardize:
        processed_data[:, np.invert(categorical)] = scale(processed_data[:, np.invert(categorical)])
        
    print('Data pre-processing finished')
    return processed_data, categorical
//  [python]
def pre_process(raw_data, categorical, impute=True, standardize=True, one_hot_encode=True):
    """
    Pre-process one dataset.

    Args:
        raw_data (np.ndarray):    raw features of the n-by-d dataset, without indices and headings.
        categorical (list):       a boolean list of length d indicating whether each raw feature is categorical.
        impute (bool):            whether to impute missing entries or not.
        standardize (bool):       whether to standardize each feature or not.
        one_hot_encode (bool):    whether to use one hot encoding to pre-process categorical features or not.
    Returns:
        np.ndarray:               pre-processed dataset.
    """
    # list of pre-processed arrays (sub-portions of dataset)
    processed = []

    # whether to impute missing entries
    if impute:
        # if there are any categorical features
        if np.array(categorical).any():
            raw_categorical = raw_data[:, categorical]
            # impute missing entries in categorical features using the most frequent number
            imp_categorical = SimpleImputer(strategy='most_frequent', copy=False)
            processed.append(imp_categorical.fit_transform(raw_categorical))

        # if there are any numeric features
        if np.invert(categorical).any():
            raw_numeric = raw_data[:, np.invert(categorical)]
            # impute missing entries in non-categorical features using mean
            imp_numeric = SimpleImputer(strategy='mean', copy=False)
            processed.append(imp_numeric.fit_transform(raw_numeric))

        # data has now been re-ordered so all categorical features appear first
        categorical = np.array(sorted(categorical, reverse=True))
        processed_data = np.hstack(tuple(processed))

    else:
        processed_data = raw_data

    # one-hot encoding for categorical features (only if there exist any)
    if one_hot_encode and np.array(categorical).any():
        categorical_columns = list(np.where(categorical)[0])
#         encoder = OneHotEncoder(categorical_features=categorical)
        encoder = ColumnTransformer([('one_hot_encoder', OneHotEncoder(), categorical_columns)], 
                        remainder='passthrough', sparse_threshold=0)
        processed_data = encoder.fit_transform(processed_data)
        categorical = np.zeros(processed_data.shape[1], dtype=bool)
    
    # standardize all numeric and one-hot encoded categorical features
    if standardize:
        processed_data[:, np.invert(categorical)] = scale(processed_data[:, np.invert(categorical)])
        
    print('Data pre-processing finished')
    return processed_data, categorical
//  [python]
def create_train_pipeline(cls, model_params: dict) -> Pipeline:

        preprocessor = ColumnTransformer(
            transformers=[
                ('numeric_transformer',
                 Imputer(strategy='median'),
                 make_column_selector(dtype_exclude='object')
                 ),
                ('categorical_transformer',
                 OneHotEncoder(handle_unknown='ignore'),
                 make_column_selector(dtype_include='object')
                 ),
            ],
            remainder='passthrough',
            sparse_threshold=0
        )

        rf_classifier = RandomForestClassifier(**model_params)

        pipeline = Pipeline([
            ('preprocessor', preprocessor),
            ('classifier', rf_classifier),
        ])

        return pipeline
//  [python]
def create_train_pipeline(cls, model_params: dict) -> Pipeline:

        preprocessor = ColumnTransformer(
            transformers=[
                ('numeric_transformer',
                 SimpleImputer(strategy='median'),
                 make_column_selector(dtype_exclude='object')
                 ),
                ('categorical_transformer',
                 OneHotEncoder(handle_unknown='ignore'),
                 make_column_selector(dtype_include='object')
                 ),
            ],
            remainder='passthrough',
            sparse_threshold=0
        )

        rf_classifier = RandomForestClassifier(**model_params)

        pipeline = Pipeline([
            ('preprocessor', preprocessor),
            ('classifier', rf_classifier),
        ])

        return pipeline
//  [python]
def _modify_categorical_features(self, x):
        # prepare data for DT
        used_features = self.features
        if self.train_only_QI:
            used_features = self.quasi_identifiers
        numeric_features = [
            f
            for f in self.features
            if f in used_features and f not in self.categorical_features
        ]
        categorical_features = [
            f for f in self.categorical_features if f in used_features
        ]
        numeric_transformer = Pipeline(
            steps=[("imputer", Imputer(strategy="constant", fill_value=0))]
        )

        categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )
        encoded = preprocessor.fit_transform(x)
        return encoded
//  [python]
def _modify_categorical_features(self, x):
        # prepare data for DT
        used_features = self.features
        if self.train_only_QI:
            used_features = self.quasi_identifiers
        numeric_features = [
            f
            for f in self.features
            if f in used_features and f not in self.categorical_features
        ]
        categorical_features = [
            f for f in self.categorical_features if f in used_features
        ]
        numeric_transformer = Pipeline(
            steps=[("imputer", SimpleImputer(strategy="constant", fill_value=0))]
        )

        categorical_transformer = OneHotEncoder(handle_unknown="ignore", sparse=False)
        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )
        encoded = preprocessor.fit_transform(x)
        return encoded
//  [python]
def ProcessData():
    # read data
    train_df = pd.read_excel("train.xlsx")
    test_df = pd.read_excel("test.xlsx")

    # emerage data
    full_df = pd.concat([train_df, test_df])

    # time feature
    cols_to_convert = ['Average temperature [°C]', 'Average wind direction [°]', 'Maximum wind speed [m/s]']
    for col in cols_to_convert:
        full_df[col] = pd.to_numeric(full_df[col], errors='coerce')

    # missing value
    imputer = Imputer(strategy="mean")
    full_df[cols_to_convert] = imputer.fit_transform(full_df[cols_to_convert])

    # drop non-numeric features
    non_numeric_features = ['Observation station', 'Time [Local time]']
    full_df.drop(non_numeric_features, axis=1, inplace=True)

    # split data
    train = full_df[full_df['Year'] < 2024]
    test = full_df[full_df['Year'] >= 2024]

    # split features and target
    X_train = train.drop('Average temperature [°C]', axis=1)
    y_train = train['Average temperature [°C]']
    X_test = test.drop('Average temperature [°C]', axis=1)
    y_test = test['Average temperature [°C]']

    return X_train, X_test, y_train, y_test
//  [python]
def ProcessData():
    # read data
    train_df = pd.read_excel("train.xlsx")
    test_df = pd.read_excel("test.xlsx")

    # emerage data
    full_df = pd.concat([train_df, test_df])

    # time feature
    cols_to_convert = ['Average temperature [°C]', 'Average wind direction [°]', 'Maximum wind speed [m/s]']
    for col in cols_to_convert:
        full_df[col] = pd.to_numeric(full_df[col], errors='coerce')

    # missing value
    imputer = SimpleImputer(strategy="mean")
    full_df[cols_to_convert] = imputer.fit_transform(full_df[cols_to_convert])

    # drop non-numeric features
    non_numeric_features = ['Observation station', 'Time [Local time]']
    full_df.drop(non_numeric_features, axis=1, inplace=True)

    # split data
    train = full_df[full_df['Year'] < 2024]
    test = full_df[full_df['Year'] >= 2024]

    # split features and target
    X_train = train.drop('Average temperature [°C]', axis=1)
    y_train = train['Average temperature [°C]']
    X_test = test.drop('Average temperature [°C]', axis=1)
    y_test = test['Average temperature [°C]']

    return X_train, X_test, y_train, y_test
//  [python]
def model_train(ds_df, run):

    ds_df.drop("Sno", axis=1, inplace=True)

    y_raw = ds_df['Risk']
    X_raw = ds_df.drop('Risk', axis=1)

    categorical_features = X_raw.select_dtypes(include=['object']).columns
    numeric_features = X_raw.select_dtypes(include=['int64', 'float']).columns

    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='constant', fill_value="missing")),
        ('onehotencoder', OneHotEncoder(categories='auto', sparse=False))])

    numeric_transformer = Pipeline(steps=[
        ('scaler', StandardScaler())])

    feature_engineering_pipeline = ColumnTransformer(
        transformers=[
            ('numeric', numeric_transformer, numeric_features),
            ('categorical', categorical_transformer, categorical_features)
        ], remainder="drop")

    # Encode Labels
    le = LabelEncoder()
    encoded_y = le.fit_transform(y_raw)

    # Train test split
    X_train, X_test, y_train, y_test = train_test_split(X_raw, encoded_y, test_size=0.20, stratify=encoded_y, random_state=42)

    # Create sklearn pipeline
    lr_clf = Pipeline(steps=[('preprocessor', feature_engineering_pipeline),
                             ('classifier', LogisticRegression(solver="lbfgs"))])
    # Train the model
    lr_clf.fit(X_train, y_train)

    # Capture metrics
    train_acc = lr_clf.score(X_train, y_train)
    test_acc = lr_clf.score(X_test, y_test)
    print("Training accuracy: %.3f" % train_acc)
    print("Test data accuracy: %.3f" % test_acc)

    # Log to Azure ML
    run.log('Train accuracy', train_acc)
    run.log('Test accuracy', test_acc)

    return lr_clf
//  [python]
def model_train(ds_df, run):

    ds_df.drop("Sno", axis=1, inplace=True)

    y_raw = ds_df['Risk']
    X_raw = ds_df.drop('Risk', axis=1)

    categorical_features = X_raw.select_dtypes(include=['object']).columns
    numeric_features = X_raw.select_dtypes(include=['int64', 'float']).columns

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value="missing")),
        ('onehotencoder', OneHotEncoder(categories='auto', sparse=False))])

    numeric_transformer = Pipeline(steps=[
        ('scaler', StandardScaler())])

    feature_engineering_pipeline = ColumnTransformer(
        transformers=[
            ('numeric', numeric_transformer, numeric_features),
            ('categorical', categorical_transformer, categorical_features)
        ], remainder="drop")

    # Encode Labels
    le = LabelEncoder()
    encoded_y = le.fit_transform(y_raw)

    # Train test split
    X_train, X_test, y_train, y_test = train_test_split(X_raw, encoded_y, test_size=0.20, stratify=encoded_y, random_state=42)

    # Create sklearn pipeline
    lr_clf = Pipeline(steps=[('preprocessor', feature_engineering_pipeline),
                             ('classifier', LogisticRegression(solver="lbfgs"))])
    # Train the model
    lr_clf.fit(X_train, y_train)

    # Capture metrics
    train_acc = lr_clf.score(X_train, y_train)
    test_acc = lr_clf.score(X_test, y_test)
    print("Training accuracy: %.3f" % train_acc)
    print("Test data accuracy: %.3f" % test_acc)

    # Log to Azure ML
    run.log('Train accuracy', train_acc)
    run.log('Test accuracy', test_acc)

    return lr_clf
//  [python]
def __init__(self, search_space_file, X, y, seed):
        self.X = X
        self.y = y
        self.search_space = []
        self.search_space_description = json.load(open(search_space_file))
        self.seed = np.random.randint(10**9) if seed is None else seed
        self.rs = np.random.RandomState(self.seed)
        
        # build all possible structural pipelines
        choices = []
        self.step_names = []
        for step in self.search_space_description:
            self.step_names.append(step["name"])
            choice_in_step = []
            if step["name"] != "classifier":
                choice_in_step.append(None)
            choice_in_step.extend(step["components"])
            choices.append(choice_in_step)
        self.pl_choices = list(it.product(*choices))
        

        
        
        # determine fixed pre-processing steps for imputation and binarization
        types = [set([type(v) for v in r]) for r in X.T]
        numeric_features = [c for c, t in enumerate(types) if len(t) == 1 and list(t)[0] != str]
        numeric_transformer = Pipeline([("imputer", sklearn.preprocessing.Imputer(strategy="median"))])
        categorical_features = [i for i in range(X.shape[1]) if i not in numeric_features]
        missing_values_per_feature = np.sum(pd.isnull(X), axis=0)
        if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
            categorical_transformer = Pipeline([
                ("imputer", sklearn.preprocessing.Imputer(strategy="most_frequent")),
                ("binarizer", sklearn.preprocessing.OneHotEncoder(handle_unknown='ignore', sparse = True)),

            ])
            self.mandatory_pre_processing = [("impute_and_binarize", ColumnTransformer(
                transformers=[
                    ("num", numeric_transformer, numeric_features),
                    ("cat", categorical_transformer, categorical_features),
                ]
            ))]
        else:
            self.mandatory_pre_processing = []
//  [python]
def __init__(self, search_space_file, X, y, seed):
        self.X = X
        self.y = y
        self.search_space = []
        self.search_space_description = json.load(open(search_space_file))
        self.seed = np.random.randint(10**9) if seed is None else seed
        self.rs = np.random.RandomState(self.seed)
        
        # build all possible structural pipelines
        choices = []
        self.step_names = []
        for step in self.search_space_description:
            self.step_names.append(step["name"])
            choice_in_step = []
            if step["name"] != "classifier":
                choice_in_step.append(None)
            choice_in_step.extend(step["components"])
            choices.append(choice_in_step)
        self.pl_choices = list(it.product(*choices))
        

        
        
        # determine fixed pre-processing steps for imputation and binarization
        types = [set([type(v) for v in r]) for r in X.T]
        numeric_features = [c for c, t in enumerate(types) if len(t) == 1 and list(t)[0] != str]
        numeric_transformer = Pipeline([("imputer", sklearn.impute.SimpleImputer(strategy="median"))])
        categorical_features = [i for i in range(X.shape[1]) if i not in numeric_features]
        missing_values_per_feature = np.sum(pd.isnull(X), axis=0)
        if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
            categorical_transformer = Pipeline([
                ("imputer", sklearn.impute.SimpleImputer(strategy="most_frequent")),
                ("binarizer", sklearn.preprocessing.OneHotEncoder(handle_unknown='ignore', sparse = True)),

            ])
            self.mandatory_pre_processing = [("impute_and_binarize", ColumnTransformer(
                transformers=[
                    ("num", numeric_transformer, numeric_features),
                    ("cat", categorical_transformer, categorical_features),
                ]
            ))]
        else:
            self.mandatory_pre_processing = []
//  [python]
def train(data_dir,
          model_dir,
          model_type,
          num_cols,
          cat_cols,
          random_state,
          **train_params):
    X_train = pd.read_pickle(data_dir/'X_train.pkl')
    y_train = pd.read_pickle(data_dir/'y_train.pkl')
    
    if model_type == "randomforest":
        clf = RandomForestClassifier(random_state=random_state, 
                                 **train_params)
    elif model_type == "lightgbm":
        clf = LGBMClassifier(random_state=random_state, 
                                    **train_params)
    elif model_type == "xgboost":
        clf = XGBClassifier(random_state=random_state, 
                                    **train_params)

    numeric_transformer = Pipeline(
        steps=[
            ("imputer", Imputer()),
            ("scaler", StandardScaler())
            ]
        )
    categorical_transformer = OrdinalEncoder()

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, num_cols),
            ("cat", categorical_transformer, cat_cols),
        ]
    )
    model = Pipeline(
        steps=[("preprocessor", preprocessor), ("clf", clf)]
        )
    
    model.fit(X_train, y_train)
    save(
        model,
        model_dir / "clf-model",
        sample_data=X_train
    )
//  [python]
def train(data_dir,
          model_dir,
          model_type,
          num_cols,
          cat_cols,
          random_state,
          **train_params):
    X_train = pd.read_pickle(data_dir/'X_train.pkl')
    y_train = pd.read_pickle(data_dir/'y_train.pkl')
    
    if model_type == "randomforest":
        clf = RandomForestClassifier(random_state=random_state, 
                                 **train_params)
    elif model_type == "lightgbm":
        clf = LGBMClassifier(random_state=random_state, 
                                    **train_params)
    elif model_type == "xgboost":
        clf = XGBClassifier(random_state=random_state, 
                                    **train_params)

    numeric_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer()),
            ("scaler", StandardScaler())
            ]
        )
    categorical_transformer = OrdinalEncoder()

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, num_cols),
            ("cat", categorical_transformer, cat_cols),
        ]
    )
    model = Pipeline(
        steps=[("preprocessor", preprocessor), ("clf", clf)]
        )
    
    model.fit(X_train, y_train)
    save(
        model,
        model_dir / "clf-model",
        sample_data=X_train
    )
//  [python]
cat_cols = ['TEAM_ABBREVIATION_HOME','SEASON']


X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  Imputer(strategy="median"), StandardScaler())
//  [python]
cat_cols = ['TEAM_ABBREVIATION_HOME','SEASON']


X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def testRegressionMultiTaskFit(self, model_name, random_state: list[int | None]):
        """Test model training for multitask regression models."""
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.REGRESSION,
                    "imputer": Imputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_regression",
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = RandomModel(
            name=f"{model_name}_multitask_regression",
            base_dir=model.baseDir,
            alg=MedianDistributionAlgorithm
        )
        self.predictorTest(predictor, dataset)
        
        # test if the results are (not) equal if the random state is the (not) same
        # and check if the output is the same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=f"{model_name}_multitask_regression",
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor,
                dataset,
                comparison_model,
                expect_equal_result=random_state[0] == random_state[1],
            )
//  [python]
def testRegressionMultiTaskFit(self, model_name, random_state: list[int | None]):
        """Test model training for multitask regression models."""
        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.REGRESSION,
                    "imputer": SimpleImputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_regression",
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = RandomModel(
            name=f"{model_name}_multitask_regression",
            base_dir=model.baseDir,
            alg=MedianDistributionAlgorithm
        )
        self.predictorTest(predictor, dataset)
        
        # test if the results are (not) equal if the random state is the (not) same
        # and check if the output is the same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=f"{model_name}_multitask_regression",
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor,
                dataset,
                comparison_model,
                expect_equal_result=random_state[0] == random_state[1],
            )
//  [python]
def __init__(self, method: str = 'linear', fail_save: TransformerMixin = Imputer()):
        self.method = method
        self.fail_save = fail_save
//  [python]
def __init__(self, method: str = 'linear', fail_save: TransformerMixin = SimpleImputer()):
        self.method = method
        self.fail_save = fail_save
//  [python]
numeric_selector = make_column_selector(dtype_include=np.number)

# This selector tells sklearn which columns in a pd.DataFrame are categorical
# Note that it will return True for text columns as well
# This is ok, but if you don't like it, you could write a more complicated is_categorical function
categorical_selector = make_column_selector(dtype_include=object)


##############################
### Preprocessing
##############################

# For numerics we:
# 1. Impute missing values with the median, adding a missing value indicator
# 2. Then center and scale the data
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("scaler", StandardScaler()),
    ]
)
//  [python]
def preprocess_P1() -> ColumnTransformer:
    '''Gets "default" preprocessing pipeline

    Returns:
        ColumnTransformer: The pipeline
    '''
    numeric_pipeline = make_pipeline(Imputer(strategy='median'), StandardScaler())
    # cat_pipeline = make_pipeline(Imputer(strategy='most_frequent'), OneHotEncoder(handle_unknown='ignore'))
    # text_pipeline = make_pipeline(CountVectorizer(), SelectKBest(k=5))

    # Check https://scikit-learn.org/stable/modules/generated/sklearn.compose.make_column_selector.html
    # and https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.select_dtypes.html#pandas.DataFrame.select_dtypes
    # to understand make_column_selector

    # /!\
    # BE VERY CAUTIOUS WHEN USING FunctionTransformer !
    # A pickled pipeline can still depends on a module definition. Hence, even when pickled, you may not have consistent results !
    # Please try to use lambdas or local function, without any dependency. It reduces risk of changes.
    # If you still have to use a module function, try to never change it later on.
    # https://stackoverflow.com/questions/73788824/how-can-i-save-reload-a-functiontransformer-object-and-expect-it-to-always-wor
    # https://github.com/OSS-Pole-Emploi/gabarit/issues/63
    # /!\

    # /!\ EXEMPLE HERE /!\
    # Good practice: Use directly the names of the columns instead of a "selector"
    # WARNING: The text pipeline is supposed to work on a column 'text' -> Please adapt it to your project if you want to use it

    # By default, we only keep the preprocess on numerical columns
    transformers = [
        ('num', numeric_pipeline, make_column_selector(dtype_include='number')),
        # ('cat', cat_pipeline, make_column_selector(dtype_include='category')), # To convert a column in a column with dtype category: df["A"].astype("category")
        # ('text', text_pipeline, 'text'), # CountVectorizer possible one column at a time
    ]

    # TODO: add sparse compatibility !
    # Use somethings like this :
    # - After applying a pipeline ...
    # if scipy.sparse.issparse(preprocessed_x):
    #     preprocessed_df = pd.DataFrame.sparse.from_spmatrix(preprocessed_x)
    # - Before training ...
    # x_train = x_train.sparse.to_coo().tocsr()
    # x_valid = x_valid.sparse.to_coo().tocsr()
    # ...
    pipeline = ColumnTransformer(transformers, sparse_threshold=0, remainder='drop')  # Use remainder='passthrough' to keep all other columns (not recommended)

    return pipeline
//  [python]
def preprocess_P1() -> ColumnTransformer:
    '''Gets "default" preprocessing pipeline

    Returns:
        ColumnTransformer: The pipeline
    '''
    numeric_pipeline = make_pipeline(SimpleImputer(strategy='median'), StandardScaler())
    # cat_pipeline = make_pipeline(SimpleImputer(strategy='most_frequent'), OneHotEncoder(handle_unknown='ignore'))
    # text_pipeline = make_pipeline(CountVectorizer(), SelectKBest(k=5))

    # Check https://scikit-learn.org/stable/modules/generated/sklearn.compose.make_column_selector.html
    # and https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.select_dtypes.html#pandas.DataFrame.select_dtypes
    # to understand make_column_selector

    # /!\
    # BE VERY CAUTIOUS WHEN USING FunctionTransformer !
    # A pickled pipeline can still depends on a module definition. Hence, even when pickled, you may not have consistent results !
    # Please try to use lambdas or local function, without any dependency. It reduces risk of changes.
    # If you still have to use a module function, try to never change it later on.
    # https://stackoverflow.com/questions/73788824/how-can-i-save-reload-a-functiontransformer-object-and-expect-it-to-always-wor
    # https://github.com/OSS-Pole-Emploi/gabarit/issues/63
    # /!\

    # /!\ EXEMPLE HERE /!\
    # Good practice: Use directly the names of the columns instead of a "selector"
    # WARNING: The text pipeline is supposed to work on a column 'text' -> Please adapt it to your project if you want to use it

    # By default, we only keep the preprocess on numerical columns
    transformers = [
        ('num', numeric_pipeline, make_column_selector(dtype_include='number')),
        # ('cat', cat_pipeline, make_column_selector(dtype_include='category')), # To convert a column in a column with dtype category: df["A"].astype("category")
        # ('text', text_pipeline, 'text'), # CountVectorizer possible one column at a time
    ]

    # TODO: add sparse compatibility !
    # Use somethings like this :
    # - After applying a pipeline ...
    # if scipy.sparse.issparse(preprocessed_x):
    #     preprocessed_df = pd.DataFrame.sparse.from_spmatrix(preprocessed_x)
    # - Before training ...
    # x_train = x_train.sparse.to_coo().tocsr()
    # x_valid = x_valid.sparse.to_coo().tocsr()
    # ...
    pipeline = ColumnTransformer(transformers, sparse_threshold=0, remainder='drop')  # Use remainder='passthrough' to keep all other columns (not recommended)

    return pipeline
//  [python]
def testClassificationMultiTaskFit(self, _, model_name, model_class, random_state):
        """Test model training for multitask classification models."""
        parameters = {}

        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [0.3],
                    "imputer": Imputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [6.5],
                    "imputer": Imputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_classification",
            alg=model_class,
            parameters=parameters,
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = RandomModel(
            name=f"{model_name}_multitask_classification",
            base_dir=model.baseDir,
            alg=RatioDistributionAlgorithm
        )

        self.predictorTest(predictor, dataset)
        
        # test if the results are (not) equal if the random state is the (not) same
        # and check if the output is the same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=f"{model_name}_multitask_classification",
                alg=model_class,
                parameters=parameters,
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor, # model loaded from file
                dataset,
                comparison_model, # model not loaded from file
                expect_equal_result=random_state[0] == random_state[1],
            )
//  [python]
def testClassificationMultiTaskFit(self, _, model_name, model_class, random_state):
        """Test model training for multitask classification models."""
        parameters = {}

        # initialize dataset
        dataset = self.createLargeTestDataSet(
            target_props=[
                {
                    "name": "fu",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [0.3],
                    "imputer": SimpleImputer(strategy="mean"),
                },
                {
                    "name": "CL",
                    "task": TargetTasks.SINGLECLASS,
                    "th": [6.5],
                    "imputer": SimpleImputer(strategy="mean"),
                },
            ],
            preparation_settings=self.getDefaultPrep(),
        )
        # test classifier
        # initialize model for training from class
        model = self.getModel(
            name=f"{model_name}_multitask_classification",
            alg=model_class,
            parameters=parameters,
            random_state=random_state[0],
        )
        self.fitTest(model, dataset)
        predictor = RandomModel(
            name=f"{model_name}_multitask_classification",
            base_dir=model.baseDir,
            alg=RatioDistributionAlgorithm
        )

        self.predictorTest(predictor, dataset)
        
        # test if the results are (not) equal if the random state is the (not) same
        # and check if the output is the same before and after saving and loading
        if random_state[0] is not None:
            comparison_model = self.getModel(
                name=f"{model_name}_multitask_classification",
                alg=model_class,
                parameters=parameters,
                random_state=random_state[1],
            )
            self.fitTest(comparison_model, dataset)
            self.predictorTest(
                predictor, # model loaded from file
                dataset,
                comparison_model, # model not loaded from file
                expect_equal_result=random_state[0] == random_state[1],
            )
//  [python]
def data_preprocessing(df:pd.DataFrame)-> np.ndarray:
    df = df.drop(['Date_of_Surgery','Date_of_Last_Visit'],axis=1)
    numerical_columns = [column for column in df.columns if df[column].dtype in ['int64','float64']]
    categorical_columns = [column for column in df.columns if df[column].dtype == 'object' and column != 'Patient_Status']
    #print(numerical_columns)
    #print(categorical_columns)
    numerical_preprocessor = Pipeline(steps=[
    ("imputer",Imputer(strategy='mean')),
    ("scaler",StandardScaler())
    ])

    categorical_preprocessor = Pipeline(steps=[
        ("imputer",Imputer(strategy='most_frequent')),
        ("onehot",OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(
    transformers=[
        ("numerical",numerical_preprocessor,numerical_columns),
        ("categorical",categorical_preprocessor,categorical_columns)
    ])
    
    X = df.drop("Patient_Status",axis=1)
    y = df.Patient_Status
    
    X_train,X_test,y_train,y_test = train_test_split(X,
                                                y,
                                                test_size=0.2,
                                                random_state=42)
    X_train = preprocessor.fit_transform(X_train)
    X_test = preprocessor.transform(X_test)

    with open('model/pipe.pkl','wb') as file:
      pickle.dump(preprocessor,file)
      
    label_encoder = LabelEncoder()

    y_train = label_encoder.fit_transform(y_train)
    y_test = label_encoder.transform(y_test)
    
    return X_train,X_test,y_train,y_test
//  [python]
def data_preprocessing(df:pd.DataFrame)-> np.ndarray:
    df = df.drop(['Date_of_Surgery','Date_of_Last_Visit'],axis=1)
    numerical_columns = [column for column in df.columns if df[column].dtype in ['int64','float64']]
    categorical_columns = [column for column in df.columns if df[column].dtype == 'object' and column != 'Patient_Status']
    #print(numerical_columns)
    #print(categorical_columns)
    numerical_preprocessor = Pipeline(steps=[
    ("imputer",SimpleImputer(strategy='mean')),
    ("scaler",StandardScaler())
    ])

    categorical_preprocessor = Pipeline(steps=[
        ("imputer",SimpleImputer(strategy='most_frequent')),
        ("onehot",OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(
    transformers=[
        ("numerical",numerical_preprocessor,numerical_columns),
        ("categorical",categorical_preprocessor,categorical_columns)
    ])
    
    X = df.drop("Patient_Status",axis=1)
    y = df.Patient_Status
    
    X_train,X_test,y_train,y_test = train_test_split(X,
                                                y,
                                                test_size=0.2,
                                                random_state=42)
    X_train = preprocessor.fit_transform(X_train)
    X_test = preprocessor.transform(X_test)

    with open('model/pipe.pkl','wb') as file:
      pickle.dump(preprocessor,file)
      
    label_encoder = LabelEncoder()

    y_train = label_encoder.fit_transform(y_train)
    y_test = label_encoder.transform(y_test)
    
    return X_train,X_test,y_train,y_test
//  [python]
def processed_dataframes(self, *args, **kwargs) -> Dict[str, pd.DataFrame]:
        df_train = self.raw_dataframe(train=True)
        df_train = df_train.drop(columns="Weight")
        df_test = self.raw_dataframe(train=False)
        df_test = df_test.drop(columns="Weight")

        df_train, df_val = train_test_split(
            df_train, stratify=df_train[self.target_columns], **kwargs
        )

        # preprocessing
        sc = StandardScaler()
        df_train[self.continuous_columns] = sc.fit_transform(
            df_train[self.continuous_columns]
        )
        df_val[self.continuous_columns] = sc.transform(df_val[self.continuous_columns])
        df_test[self.continuous_columns] = sc.transform(
            df_test[self.continuous_columns]
        )

        df_train = self._label_encoder(df_train)
        df_val = self._label_encoder(df_val)
        df_test = self._label_encoder(df_test)

        # if value == -999.0, imputated by avg of the column
        df_train = df_train.replace("-999", np.nan)
        df_val = df_val.replace("-999", np.nan)
        df_test = df_test.replace("-999", np.nan)
        df_train = pd.DataFrame(
            data=Imputer().fit_transform(df_train),
            index=df_train.index,
            columns=df_train.columns,
        )
        df_val = pd.DataFrame(
            data=Imputer().fit_transform(df_val),
            index=df_val.index,
            columns=df_test.columns,
        )
        df_test = pd.DataFrame(
            data=Imputer().fit_transform(df_test),
            index=df_test.index,
            columns=df_test.columns,
        )

        return {
            "train": df_train,
            "val": df_val,
            "test": df_test,
        }
//  [python]
def processed_dataframes(self, *args, **kwargs) -> Dict[str, pd.DataFrame]:
        df_train = self.raw_dataframe(train=True)
        df_train = df_train.drop(columns="Weight")
        df_test = self.raw_dataframe(train=False)
        df_test = df_test.drop(columns="Weight")

        df_train, df_val = train_test_split(
            df_train, stratify=df_train[self.target_columns], **kwargs
        )

        # preprocessing
        sc = StandardScaler()
        df_train[self.continuous_columns] = sc.fit_transform(
            df_train[self.continuous_columns]
        )
        df_val[self.continuous_columns] = sc.transform(df_val[self.continuous_columns])
        df_test[self.continuous_columns] = sc.transform(
            df_test[self.continuous_columns]
        )

        df_train = self._label_encoder(df_train)
        df_val = self._label_encoder(df_val)
        df_test = self._label_encoder(df_test)

        # if value == -999.0, imputated by avg of the column
        df_train = df_train.replace("-999", np.nan)
        df_val = df_val.replace("-999", np.nan)
        df_test = df_test.replace("-999", np.nan)
        df_train = pd.DataFrame(
            data=SimpleImputer().fit_transform(df_train),
            index=df_train.index,
            columns=df_train.columns,
        )
        df_val = pd.DataFrame(
            data=SimpleImputer().fit_transform(df_val),
            index=df_val.index,
            columns=df_test.columns,
        )
        df_test = pd.DataFrame(
            data=SimpleImputer().fit_transform(df_test),
            index=df_test.index,
            columns=df_test.columns,
        )

        return {
            "train": df_train,
            "val": df_val,
            "test": df_test,
        }
//  [python]
def test_fit(data):
    a = sklearn.preprocessing.Imputer()
    b = dask_ml.impute.SimpleImputer()

    a.fit(X)
    b.fit(data)

    assert_estimator_equal(a, b, exclude=["statistics_"])
    np.testing.assert_array_almost_equal(a.statistics_, np.asarray(b.statistics_))
//  [python]
def test_fit(data):
    a = sklearn.impute.SimpleImputer()
    b = dask_ml.impute.SimpleImputer()

    a.fit(X)
    b.fit(data)

    assert_estimator_equal(a, b, exclude=["statistics_"])
    np.testing.assert_array_almost_equal(a.statistics_, np.asarray(b.statistics_))