jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def load_data(self, file_name: str) -> pd.DataFrame:
        """Load dataset.

        Args:
            file_name: path of the dataset.

        Returns:
            Dataframe with the loaded dataset.
        """
        feature_eng = Features(formula_file=file_name)
        features = feature_eng.get_features()
        df = pd.DataFrame(features)
        df = df.dropna()

        imp = Imputer(missing_values=np.nan, strategy="mean")
        imp.fit(df.values.tolist())
        data_list = imp.transform(df.values.tolist())
        df = pd.DataFrame(data_list)

        if self.crystal_sys == "monoclinic":
            # Selecting only the monoclinic materials
            df = df.loc[df[1] == 1]
        elif self.crystal_sys == "triclinic":
            # Selecting only the triclinic materials
            df = df.loc[df[2] == 1]
        elif self.crystal_sys == "orthorhombic":
            # Selecting only the orthorhombic materials
            df = df.loc[df[3] == 1]
        elif self.crystal_sys == "trigonal":
            # Selecting only the trigonal materials
            df = df.loc[df[4] == 1]
        elif self.crystal_sys == "hexagonal":
            # Selecting only the hexagonal materials
            df = df.loc[df[5] == 1]
        elif self.crystal_sys == "cubic":
            # Selecting only the cubic materials
            df = df.loc[df[6] == 1]
        elif self.crystal_sys == "tetragonal":
            # Selecting only the tetragonal materials
            df = df.loc[df[7] == 1]
        elif self.crystal_sys == "all":
            # Selecting  the all materials
            pass

        return df
//  [python]
def load_data(self, file_name: str) -> pd.DataFrame:
        """Load dataset.

        Args:
            file_name: path of the dataset.

        Returns:
            Dataframe with the loaded dataset.
        """
        feature_eng = Features(formula_file=file_name)
        features = feature_eng.get_features()
        df = pd.DataFrame(features)
        df = df.dropna()

        imp = SimpleImputer(missing_values=np.nan, strategy="mean")
        imp.fit(df.values.tolist())
        data_list = imp.transform(df.values.tolist())
        df = pd.DataFrame(data_list)

        if self.crystal_sys == "monoclinic":
            # Selecting only the monoclinic materials
            df = df.loc[df[1] == 1]
        elif self.crystal_sys == "triclinic":
            # Selecting only the triclinic materials
            df = df.loc[df[2] == 1]
        elif self.crystal_sys == "orthorhombic":
            # Selecting only the orthorhombic materials
            df = df.loc[df[3] == 1]
        elif self.crystal_sys == "trigonal":
            # Selecting only the trigonal materials
            df = df.loc[df[4] == 1]
        elif self.crystal_sys == "hexagonal":
            # Selecting only the hexagonal materials
            df = df.loc[df[5] == 1]
        elif self.crystal_sys == "cubic":
            # Selecting only the cubic materials
            df = df.loc[df[6] == 1]
        elif self.crystal_sys == "tetragonal":
            # Selecting only the tetragonal materials
            df = df.loc[df[7] == 1]
        elif self.crystal_sys == "all":
            # Selecting  the all materials
            pass

        return df
//  [python]
def impute_lab(lab_data):
    imputer = Imputer(strategy="mean")
    lab_data_impute = lab_data.copy()
    imputer.fit( lab_data.reshape((-1,lab_data.shape[1])) )
    for i,patient in enumerate(lab_data):
        for j,signal in enumerate(patient):
            nan_arr , nan_count = check_nan(signal)
            if nan_count!=len(signal):
                lab_data_impute[i,j,:] = forward_impute(signal, nan_arr)
    lab_data_impute = np.array( [imputer.transform(sample.T).T for sample in lab_data_impute] )
    return lab_data_impute
//  [python]
def impute_lab(lab_data):
    imputer = SimpleImputer(strategy="mean")
    lab_data_impute = lab_data.copy()
    imputer.fit( lab_data.reshape((-1,lab_data.shape[1])) )
    for i,patient in enumerate(lab_data):
        for j,signal in enumerate(patient):
            nan_arr , nan_count = check_nan(signal)
            if nan_count!=len(signal):
                lab_data_impute[i,j,:] = forward_impute(signal, nan_arr)
    lab_data_impute = np.array( [imputer.transform(sample.T).T for sample in lab_data_impute] )
    return lab_data_impute
//  [python]
def build_preprocessing_pipeline(
    categorical_encoding: str,
    continuous_features: List[str],
    discrete_features: List[str],
) -> ColumnTransformer:
    # Configure the categorical encoder
    if categorical_encoding == "ordinal":
        categorical_encoder = OrdinalEncoder(
            handle_unknown="use_encoded_value", unknown_value=np.nan
        )
    elif categorical_encoding == "onehot":
        categorical_encoder = OneHotEncoder(handle_unknown="ignore")
    else:
        raise ValueError(
            f"categorical_encoding '{categorical_encoding}' is not a valid encoding strategy. Possible values are 'ordinal' or 'onehot'"
        )

    # For continuous variables, replace missing values with the median and then normalize by
    # subtracting the mean and dividing by the standard deviation
    continuous_pipeline = sklearn.pipeline.Pipeline(
        [("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
    )

    # For discrete variables, encode the data
    discrete_pipeline = sklearn.pipeline.Pipeline(
        [
            ("imputer", Imputer(strategy="most_frequent")),
            ("encoder", categorical_encoder),
        ]
    )

    # Build the pipeline
    transformations = ColumnTransformer(
        [
            ("continuous_pipe", continuous_pipeline, continuous_features),
            ("discrete_pipe", discrete_pipeline, discrete_features),
        ],
        remainder="passthrough",
    )  # Target will passthrough if there

    return transformations
//  [python]
def build_preprocessing_pipeline(
    categorical_encoding: str,
    continuous_features: List[str],
    discrete_features: List[str],
) -> ColumnTransformer:
    # Configure the categorical encoder
    if categorical_encoding == "ordinal":
        categorical_encoder = OrdinalEncoder(
            handle_unknown="use_encoded_value", unknown_value=np.nan
        )
    elif categorical_encoding == "onehot":
        categorical_encoder = OneHotEncoder(handle_unknown="ignore")
    else:
        raise ValueError(
            f"categorical_encoding '{categorical_encoding}' is not a valid encoding strategy. Possible values are 'ordinal' or 'onehot'"
        )

    # For continuous variables, replace missing values with the median and then normalize by
    # subtracting the mean and dividing by the standard deviation
    continuous_pipeline = sklearn.pipeline.Pipeline(
        [("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
    )

    # For discrete variables, encode the data
    discrete_pipeline = sklearn.pipeline.Pipeline(
        [
            ("imputer", SimpleImputer(strategy="most_frequent")),
            ("encoder", categorical_encoder),
        ]
    )

    # Build the pipeline
    transformations = ColumnTransformer(
        [
            ("continuous_pipe", continuous_pipeline, continuous_features),
            ("discrete_pipe", discrete_pipeline, discrete_features),
        ],
        remainder="passthrough",
    )  # Target will passthrough if there

    return transformations
//  [python]
def _preprocess_train(self, X, feature_types, vect_max_features):
        transformer_list = []
        if feature_types.get('language', None):
            pipeline = Pipeline(steps=[
                ("preparator", NlpDataPreprocessor(nlp_cols=feature_types['language'])),
                ("vectorizer", TfidfVectorizer(ngram_range=self.params['proc.ngram_range'], sublinear_tf=True, max_features=vect_max_features, tokenizer=self._tokenize)),
            ])
            transformer_list.append(('vect', pipeline, feature_types['language']))
        if feature_types.get('onehot', None):
            pipeline = Pipeline(steps=[
                ('generator', OheFeaturesGenerator()),
            ])
            transformer_list.append(('cats', pipeline, feature_types['onehot']))
        if feature_types.get('continuous', None):
            pipeline = Pipeline(steps=[
                ('imputer', Imputer(strategy=self.params['proc.impute_strategy'])),
                ('scaler', StandardScaler())
            ])
            transformer_list.append(('cont', pipeline, feature_types['continuous']))
        if feature_types.get('bool', None):
            pipeline = Pipeline(steps=[
                ('scaler', StandardScaler())
            ])
            transformer_list.append(('bool', pipeline, feature_types['bool']))
        if feature_types.get('skewed', None):
            pipeline = Pipeline(steps=[
                ('imputer', Imputer(strategy=self.params['proc.impute_strategy'])),
                ('quantile', QuantileTransformer(output_distribution='normal')),  # Or output_distribution = 'uniform'
            ])
            transformer_list.append(('skew', pipeline, feature_types['skewed']))
        self._pipeline = ColumnTransformer(transformers=transformer_list)
        return self._pipeline.fit_transform(X)
//  [python]
def _preprocess_train(self, X, feature_types, vect_max_features):
        transformer_list = []
        if feature_types.get('language', None):
            pipeline = Pipeline(steps=[
                ("preparator", NlpDataPreprocessor(nlp_cols=feature_types['language'])),
                ("vectorizer", TfidfVectorizer(ngram_range=self.params['proc.ngram_range'], sublinear_tf=True, max_features=vect_max_features, tokenizer=self._tokenize)),
            ])
            transformer_list.append(('vect', pipeline, feature_types['language']))
        if feature_types.get('onehot', None):
            pipeline = Pipeline(steps=[
                ('generator', OheFeaturesGenerator()),
            ])
            transformer_list.append(('cats', pipeline, feature_types['onehot']))
        if feature_types.get('continuous', None):
            pipeline = Pipeline(steps=[
                ('imputer', SimpleImputer(strategy=self.params['proc.impute_strategy'])),
                ('scaler', StandardScaler())
            ])
            transformer_list.append(('cont', pipeline, feature_types['continuous']))
        if feature_types.get('bool', None):
            pipeline = Pipeline(steps=[
                ('scaler', StandardScaler())
            ])
            transformer_list.append(('bool', pipeline, feature_types['bool']))
        if feature_types.get('skewed', None):
            pipeline = Pipeline(steps=[
                ('imputer', SimpleImputer(strategy=self.params['proc.impute_strategy'])),
                ('quantile', QuantileTransformer(output_distribution='normal')),  # Or output_distribution = 'uniform'
            ])
            transformer_list.append(('skew', pipeline, feature_types['skewed']))
        self._pipeline = ColumnTransformer(transformers=transformer_list)
        return self._pipeline.fit_transform(X)
//  [python]
def impute_missing_values(
    X_train: pd.DataFrame,
    X_val: pd.DataFrame,
    X_test: pd.DataFrame,
    strategy: str = "mean",
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    """
    Imputes missing values in all dataframes.
    """
    logging.info(f"Imputing missing values with {strategy}.")

    for column in X_train.columns:
        if X_train[column].isnull().all():
            X_train.drop(column, axis=1, inplace=True)
            X_val.drop(column, axis=1, inplace=True)
            X_test.drop(column, axis=1, inplace=True)

    imputer = Imputer(strategy=strategy)
    imputer.fit(X_train)

    X_train = pd.DataFrame(
        imputer.fit_transform(X_train), columns=X_train.columns, index=X_train.index
    )
    X_val = pd.DataFrame(
        imputer.transform(X_val), columns=X_val.columns, index=X_val.index
    )
    X_test = pd.DataFrame(
        imputer.transform(X_test), columns=X_test.columns, index=X_test.index
    )

    return X_train, X_val, X_test
//  [python]
def impute_missing_values(
    X_train: pd.DataFrame,
    X_val: pd.DataFrame,
    X_test: pd.DataFrame,
    strategy: str = "mean",
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    """
    Imputes missing values in all dataframes.
    """
    logging.info(f"Imputing missing values with {strategy}.")

    for column in X_train.columns:
        if X_train[column].isnull().all():
            X_train.drop(column, axis=1, inplace=True)
            X_val.drop(column, axis=1, inplace=True)
            X_test.drop(column, axis=1, inplace=True)

    imputer = SimpleImputer(strategy=strategy)
    imputer.fit(X_train)

    X_train = pd.DataFrame(
        imputer.fit_transform(X_train), columns=X_train.columns, index=X_train.index
    )
    X_val = pd.DataFrame(
        imputer.transform(X_val), columns=X_val.columns, index=X_val.index
    )
    X_test = pd.DataFrame(
        imputer.transform(X_test), columns=X_test.columns, index=X_test.index
    )

    return X_train, X_val, X_test
//  [python]
def train_preprocessor(data):
    """Train preprocessor."""
    print("Training preprocessor.")
    # TODO: ask if we need np.random.seed(...) here

    ordinal_features = [
        n for (n, _) in enumerate(data.feature_names)
        if n not in data.category_map
    ]

    categorical_features = list(data.category_map.keys())
    ordinal_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('scaler', StandardScaler())
    ])

    categorical_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='median')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(transformers=[
        ('num', ordinal_transformer, ordinal_features),
        ('cat', categorical_transformer, categorical_features)
    ])

    preprocessor.fit(data.data)

    return  preprocessor
//  [python]
def train_preprocessor(data):
    """Train preprocessor."""
    print("Training preprocessor.")
    # TODO: ask if we need np.random.seed(...) here

    ordinal_features = [
        n for (n, _) in enumerate(data.feature_names)
        if n not in data.category_map
    ]

    categorical_features = list(data.category_map.keys())
    ordinal_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
    ])

    categorical_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(transformers=[
        ('num', ordinal_transformer, ordinal_features),
        ('cat', categorical_transformer, categorical_features)
    ])

    preprocessor.fit(data.data)

    return  preprocessor
//  [python]
def get_data_transformation(self):
        
        try:
            logging.info('Data Transformation initiated')

            numerical_cols = ['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs', 'restecg', 'thalach', 'exang', 'oldpeak', 'slope', 'ca', 'thal']
            logging.info('Numerical Pipeline Initiated')
            
            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',Imputer(strategy='median')),
                ('scaler',StandardScaler())])

            preprocessor = ColumnTransformer([('num_pipeline',num_pipeline,numerical_cols)])
            return preprocessor 
                
        except Exception as e:
            logging.info("Exception occured in the initiate_datatransformation")
            raise customexception(e,sys)
//  [python]
def get_data_transformation(self):
        
        try:
            logging.info('Data Transformation initiated')

            numerical_cols = ['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs', 'restecg', 'thalach', 'exang', 'oldpeak', 'slope', 'ca', 'thal']
            logging.info('Numerical Pipeline Initiated')
            
            ## Numerical Pipeline
            num_pipeline=Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='median')),
                ('scaler',StandardScaler())])

            preprocessor = ColumnTransformer([('num_pipeline',num_pipeline,numerical_cols)])
            return preprocessor 
                
        except Exception as e:
            logging.info("Exception occured in the initiate_datatransformation")
            raise customexception(e,sys)
//  [python]
def get_pipeline_model_uri():
    """
    Create a pipeline model that transforms and trains on the dataset returned by
    `get_pipeline_model_dataset`. The pipeline model imputes the missing values in
    input dataset, encodes categorical features, and then trains a logistic regression
    model.
    """
    data, target_col = get_pipeline_model_dataset()
    X = data.drop(target_col, axis=1)
    y = data[target_col].to_numpy()

    encoder = sklearn.preprocessing.OrdinalEncoder()
    str_imputer = sklearn.preprocessing.Imputer(missing_values=None, strategy="most_frequent")
    num_imputer = sklearn.preprocessing.Imputer(missing_values=np.nan, strategy="mean")
    preproc_pipeline = sklearn.pipeline.Pipeline(
        [
            ("imputer", str_imputer),
            ("encoder", encoder),
        ]
    )

    pipeline = sklearn.pipeline.Pipeline(
        [
            (
                "transformer",
                sklearn.compose.make_column_transformer(
                    (preproc_pipeline, ["f1", "f2"]),
                    (num_imputer, ["f3", "f4"]),
                ),
            ),
            ("clf", sklearn.linear_model.LogisticRegression()),
        ]
    )
    pipeline.fit(X, y)

    with mlflow.start_run():
        model_info = mlflow.sklearn.log_model(pipeline, "pipeline_model")
        return model_info.model_uri
//  [python]
def get_pipeline_model_uri():
    """
    Create a pipeline model that transforms and trains on the dataset returned by
    `get_pipeline_model_dataset`. The pipeline model imputes the missing values in
    input dataset, encodes categorical features, and then trains a logistic regression
    model.
    """
    data, target_col = get_pipeline_model_dataset()
    X = data.drop(target_col, axis=1)
    y = data[target_col].to_numpy()

    encoder = sklearn.preprocessing.OrdinalEncoder()
    str_imputer = sklearn.impute.SimpleImputer(missing_values=None, strategy="most_frequent")
    num_imputer = sklearn.impute.SimpleImputer(missing_values=np.nan, strategy="mean")
    preproc_pipeline = sklearn.pipeline.Pipeline(
        [
            ("imputer", str_imputer),
            ("encoder", encoder),
        ]
    )

    pipeline = sklearn.pipeline.Pipeline(
        [
            (
                "transformer",
                sklearn.compose.make_column_transformer(
                    (preproc_pipeline, ["f1", "f2"]),
                    (num_imputer, ["f3", "f4"]),
                ),
            ),
            ("clf", sklearn.linear_model.LogisticRegression()),
        ]
    )
    pipeline.fit(X, y)

    with mlflow.start_run():
        model_info = mlflow.sklearn.log_model(pipeline, "pipeline_model")
        return model_info.model_uri
//  [python]
def save_cell_graph(genes_path, save_path):
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    exp = pd.read_csv(os.path.join(genes_path, 'exp.csv'), index_col=0)
    cn = pd.read_csv(os.path.join(genes_path, 'cn.csv'), index_col=0)
    mu = pd.read_csv(os.path.join(genes_path, 'mu.csv'), index_col=0)
    # me = pd.read_csv(os.path.join(genes_path, 'me.csv'), index_col=0)
    # print('Miss values：{}，{}，{}, {}'.format(exp.isna().sum().sum(), cn.isna().sum().sum(), mu.isna().sum().sum(),
    #                                         me.isna().sum().sum()))

    index = exp.index
    columns = exp.columns

    scaler = StandardScaler()
    exp = scaler.fit_transform(exp)
    cn = scaler.fit_transform(cn)
    # me = scaler.fit_transform(me)

    imp_mean = Imputer(strategy='mean')
    exp = imp_mean.fit_transform(exp)

    exp = pd.DataFrame(exp, index=index, columns=columns)
    cn = pd.DataFrame(cn, index=index, columns=columns)
    mu = pd.DataFrame(mu, index=index, columns=columns)
    # me = pd.DataFrame(me, index=index, columns=columns)
    cell_names = exp.index
    # print('Miss values：{}，{}，{}, {}'.format(exp.isna().sum().sum(), cn.isna().sum().sum(), mu.isna().sum().sum(),
    #                                         me.isna().sum().sum()))

    cell_dict = {}
    for i in cell_names:
        # cell_dict[i] = Data(x=torch.tensor([exp.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([cn.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([mu.loc[i]], dtype=torch.float).T)
        cell_dict[i] = Data(x=torch.tensor([exp.loc[i], cn.loc[i], mu.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([exp.loc[i], cn.loc[i], mu.loc[i], me.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = [np.array(exp.loc[i], dtype=np.float32), np.array(cn.loc[i], dtype=np.float32),
        #                 np.array(mu.loc[i], dtype=np.float32)]

    np.save(os.path.join(save_path, 'cell_feature_cn_std.npy'), cell_dict)
    print("finish saving cell mut data!")
//  [python]
def save_cell_graph(genes_path, save_path):
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    exp = pd.read_csv(os.path.join(genes_path, 'exp.csv'), index_col=0)
    cn = pd.read_csv(os.path.join(genes_path, 'cn.csv'), index_col=0)
    mu = pd.read_csv(os.path.join(genes_path, 'mu.csv'), index_col=0)
    # me = pd.read_csv(os.path.join(genes_path, 'me.csv'), index_col=0)
    # print('Miss values：{}，{}，{}, {}'.format(exp.isna().sum().sum(), cn.isna().sum().sum(), mu.isna().sum().sum(),
    #                                         me.isna().sum().sum()))

    index = exp.index
    columns = exp.columns

    scaler = StandardScaler()
    exp = scaler.fit_transform(exp)
    cn = scaler.fit_transform(cn)
    # me = scaler.fit_transform(me)

    imp_mean = SimpleImputer()
    exp = imp_mean.fit_transform(exp)

    exp = pd.DataFrame(exp, index=index, columns=columns)
    cn = pd.DataFrame(cn, index=index, columns=columns)
    mu = pd.DataFrame(mu, index=index, columns=columns)
    # me = pd.DataFrame(me, index=index, columns=columns)
    cell_names = exp.index
    # print('Miss values：{}，{}，{}, {}'.format(exp.isna().sum().sum(), cn.isna().sum().sum(), mu.isna().sum().sum(),
    #                                         me.isna().sum().sum()))

    cell_dict = {}
    for i in cell_names:
        # cell_dict[i] = Data(x=torch.tensor([exp.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([cn.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([mu.loc[i]], dtype=torch.float).T)
        cell_dict[i] = Data(x=torch.tensor([exp.loc[i], cn.loc[i], mu.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = Data(x=torch.tensor([exp.loc[i], cn.loc[i], mu.loc[i], me.loc[i]], dtype=torch.float).T)
        # cell_dict[i] = [np.array(exp.loc[i], dtype=np.float32), np.array(cn.loc[i], dtype=np.float32),
        #                 np.array(mu.loc[i], dtype=np.float32)]

    np.save(os.path.join(save_path, 'cell_feature_cn_std.npy'), cell_dict)
    print("finish saving cell mut data!")
//  [python]
dataset = pd.read_csv("cali_renaw.csv")

#checking missing columns
missing_cols = [col for col in dataset.columns if dataset[col].isna().any() == True]

#Ratios column ratios
for i in missing_cols:
    print('The ratio of missing value at col ', i, ' is: ', dataset[i].isna().sum() / dataset.shape[0])
    
#Dropping the SOLAR because more than 65 percent of it is NaN
dataset =  dataset.drop('SOLAR', axis=1)   

#imputing the missing values
imp_mean = Imputer(missing_values=np.nan, strategy='mean')
//  [python]
dataset = pd.read_csv("cali_renaw.csv")

#checking missing columns
missing_cols = [col for col in dataset.columns if dataset[col].isna().any() == True]

#Ratios column ratios
for i in missing_cols:
    print('The ratio of missing value at col ', i, ' is: ', dataset[i].isna().sum() / dataset.shape[0])
    
#Dropping the SOLAR because more than 65 percent of it is NaN
dataset =  dataset.drop('SOLAR', axis=1)   

#imputing the missing values
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
//  [python]
def model_train(credit_data_df):
    #credit_data_df = pd.read_csv("dataset/german_credit_data.csv")  # , nrows=200000, parse_dates=["LEG1_DEP_DATE_GMT", "LEG1_ARR_DATE_GMT","LEG2_DEP_DATE_GMT", "LEG2_ARR_DATE_GMT"])
    credit_data_df.drop("Sno", axis=1, inplace=True)

    y_raw = credit_data_df['Risk']
    X_raw = credit_data_df.drop('Risk', axis=1)
    #del credit_data_df

    categorical_features = X_raw.select_dtypes(include=['object']).columns
    numeric_features = X_raw.select_dtypes(include=['int64', 'float']).columns

    categorical_transformer = Pipeline(steps=[
        ('imputer', Imputer(strategy='constant', fill_value="missing")),
        ('onehotencoder', OneHotEncoder(categories='auto', sparse=False))])

    numeric_transformer = Pipeline(steps=[
        # ('imputer', Imputer(strategy='median')),
        ('scaler', StandardScaler())])

    feature_engineering_pipeline = ColumnTransformer(
        transformers=[
            ('numeric', numeric_transformer, numeric_features),
            ('categorical', categorical_transformer, categorical_features)
        ], remainder="drop")

    #Encode Labels
    le = LabelEncoder()
    encoded_y = le.fit_transform(y_raw)

    #Train test split
    X_train, X_test, y_train, y_test = train_test_split(X_raw, encoded_y, test_size=0.20, stratify=encoded_y, random_state=42)

    #Create sklearn pipeline
    lr_clf = Pipeline(steps=[('preprocessor', feature_engineering_pipeline),
                             ('classifier', LogisticRegression(solver="lbfgs"))])
    #Train the model
    lr_clf.fit(X_train, y_train)

    #Capture metrics
    train_acc = lr_clf.score(X_train, y_train)
    test_acc = lr_clf.score(X_test, y_test)
    print("training accuracy: %.3f" % train_acc)
    print("test data accuracy: %.3f" % test_acc)

    return lr_clf
//  [python]
def model_train(credit_data_df):
    #credit_data_df = pd.read_csv("dataset/german_credit_data.csv")  # , nrows=200000, parse_dates=["LEG1_DEP_DATE_GMT", "LEG1_ARR_DATE_GMT","LEG2_DEP_DATE_GMT", "LEG2_ARR_DATE_GMT"])
    credit_data_df.drop("Sno", axis=1, inplace=True)

    y_raw = credit_data_df['Risk']
    X_raw = credit_data_df.drop('Risk', axis=1)
    #del credit_data_df

    categorical_features = X_raw.select_dtypes(include=['object']).columns
    numeric_features = X_raw.select_dtypes(include=['int64', 'float']).columns

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value="missing")),
        ('onehotencoder', OneHotEncoder(categories='auto', sparse=False))])

    numeric_transformer = Pipeline(steps=[
        # ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    feature_engineering_pipeline = ColumnTransformer(
        transformers=[
            ('numeric', numeric_transformer, numeric_features),
            ('categorical', categorical_transformer, categorical_features)
        ], remainder="drop")

    #Encode Labels
    le = LabelEncoder()
    encoded_y = le.fit_transform(y_raw)

    #Train test split
    X_train, X_test, y_train, y_test = train_test_split(X_raw, encoded_y, test_size=0.20, stratify=encoded_y, random_state=42)

    #Create sklearn pipeline
    lr_clf = Pipeline(steps=[('preprocessor', feature_engineering_pipeline),
                             ('classifier', LogisticRegression(solver="lbfgs"))])
    #Train the model
    lr_clf.fit(X_train, y_train)

    #Capture metrics
    train_acc = lr_clf.score(X_train, y_train)
    test_acc = lr_clf.score(X_test, y_test)
    print("training accuracy: %.3f" % train_acc)
    print("test data accuracy: %.3f" % test_acc)

    return lr_clf
//  [python]
def fit_model(self, df, vars_to_exclude, outcome, weights=None, n_jobs=1, **kw):
        '''Fit random forest model.

        Args:
            df (pd.DataFrame): data rows, subset to training rows
            vars_to_exclude (list): variables to disallow from the model
            outcome (string): name of the column that we're predicting
            weights (list): sample weights to use for training (can be None)
            n_jobs (int): number of jobs to run in parallel

        Returns:
                tuple: tuple containing:

                - **mod** (*sklearn.ensemble.RandomForestClassifier*): trained sklearn random forest model object
                - **feature_info** (*pd.DataFrame*): feature_importance
        '''

        if df[outcome].nunique() == 1:
            logger.error('Something went wrong; all training outcomes are {}'.format(df[outcome].tolist()[0]))
            raise ValueError

        col_names = df.columns.values.tolist()
        col_names = [col for col in col_names if col not in vars_to_exclude]
        numeric_cols = [c for c in col_names if 'var_' in c and '_missing' not in c and 'match' not in c]
        categorical_cols = [c for c in col_names if 'var_' in c  and ('_missing' in c or 'match' in c)]

        numeric_transformer = FeatureUnion([
            ('impute_numeric', Pipeline(steps=[
                ('imputer', Imputer(strategy='mean'))])),
            ('missing_dummies', Pipeline(steps=[
                ('missing', MissingIndicator(error_on_new=False))]))
        ])

        categorical_transformer = Pipeline(steps=[
            ('imputer', Imputer(strategy='constant', fill_value=-1)),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_cols),
                ('cat', categorical_transformer, categorical_cols)],
            sparse_threshold=0)

        base_clf = RandomForestClassifier(
                n_estimators=200,
                oob_score=True,
                n_jobs=n_jobs,
                verbose=0)

        pipeline = Pipeline([
            ('preprocessor', preprocessor),
            ('no_variance_filter', VarianceThreshold()),
            ('clf', base_clf)
        ])

        pipeline_params = {
            'clf__min_samples_leaf':[25, 150],
        }

        mod = GridSearchCV(pipeline, pipeline_params,
			cv=3,
			n_jobs=1, verbose=0,
			scoring='f1',
            return_train_score=True
        )

        mod.fit(df, df[outcome],
                clf__sample_weight=weights)

        feature_info = get_feature_info(mod.best_estimator_, numeric_cols, categorical_cols)

        return mod, feature_info
//  [python]
def fit_model(self, df, vars_to_exclude, outcome, weights=None, n_jobs=1, **kw):
        '''Fit random forest model.

        Args:
            df (pd.DataFrame): data rows, subset to training rows
            vars_to_exclude (list): variables to disallow from the model
            outcome (string): name of the column that we're predicting
            weights (list): sample weights to use for training (can be None)
            n_jobs (int): number of jobs to run in parallel

        Returns:
                tuple: tuple containing:

                - **mod** (*sklearn.ensemble.RandomForestClassifier*): trained sklearn random forest model object
                - **feature_info** (*pd.DataFrame*): feature_importance
        '''

        if df[outcome].nunique() == 1:
            logger.error('Something went wrong; all training outcomes are {}'.format(df[outcome].tolist()[0]))
            raise ValueError

        col_names = df.columns.values.tolist()
        col_names = [col for col in col_names if col not in vars_to_exclude]
        numeric_cols = [c for c in col_names if 'var_' in c and '_missing' not in c and 'match' not in c]
        categorical_cols = [c for c in col_names if 'var_' in c  and ('_missing' in c or 'match' in c)]

        numeric_transformer = FeatureUnion([
            ('impute_numeric', Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='mean'))])),
            ('missing_dummies', Pipeline(steps=[
                ('missing', MissingIndicator(error_on_new=False))]))
        ])

        categorical_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='constant', fill_value=-1)),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_cols),
                ('cat', categorical_transformer, categorical_cols)],
            sparse_threshold=0)

        base_clf = RandomForestClassifier(
                n_estimators=200,
                oob_score=True,
                n_jobs=n_jobs,
                verbose=0)

        pipeline = Pipeline([
            ('preprocessor', preprocessor),
            ('no_variance_filter', VarianceThreshold()),
            ('clf', base_clf)
        ])

        pipeline_params = {
            'clf__min_samples_leaf':[25, 150],
        }

        mod = GridSearchCV(pipeline, pipeline_params,
			cv=3,
			n_jobs=1, verbose=0,
			scoring='f1',
            return_train_score=True
        )

        mod.fit(df, df[outcome],
                clf__sample_weight=weights)

        feature_info = get_feature_info(mod.best_estimator_, numeric_cols, categorical_cols)

        return mod, feature_info
//  [python]
def make_anchor_tabular_income(dirname: Optional[Path] = None) -> AnchorTabular:
    # adapted from:
    # https://docs.seldon.io/projects/alibi/en/latest/examples/anchor_tabular_adult.html
    np.random.seed(0)

    # prepare data
    adult = fetch_adult()
    data = adult.data
    target = adult.target
    feature_names = adult.feature_names
    category_map = adult.category_map

    data_perm = np.random.permutation(np.c_[data, target])
    data = data_perm[:, :-1]
    target = data_perm[:, -1]

    # build model
    idx = 30000
    X_train, Y_train = data[:idx, :], target[:idx]
    X_test, Y_test = data[idx + 1 :, :], target[idx + 1 :]

    ordinal_features = [
        x for x in range(len(feature_names)) if x not in list(category_map.keys())
    ]
    ordinal_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = list(category_map.keys())
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="median")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", ordinal_transformer, ordinal_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    clf = RandomForestClassifier(n_estimators=50)

    model_pipeline = Pipeline(
        steps=[
            ("preprocess", preprocessor),
            ("classifier", clf),
        ]
    )

    model_pipeline.fit(X_train, Y_train)

    explainer = AnchorTabular(
        model_pipeline.predict, feature_names, categorical_names=category_map, seed=1
    )

    explainer.fit(X_train, disc_perc=[25, 50, 75])

    if dirname is not None:
        explainer.save(dirname)
    return explainer
//  [python]
def make_anchor_tabular_income(dirname: Optional[Path] = None) -> AnchorTabular:
    # adapted from:
    # https://docs.seldon.io/projects/alibi/en/latest/examples/anchor_tabular_adult.html
    np.random.seed(0)

    # prepare data
    adult = fetch_adult()
    data = adult.data
    target = adult.target
    feature_names = adult.feature_names
    category_map = adult.category_map

    data_perm = np.random.permutation(np.c_[data, target])
    data = data_perm[:, :-1]
    target = data_perm[:, -1]

    # build model
    idx = 30000
    X_train, Y_train = data[:idx, :], target[:idx]
    X_test, Y_test = data[idx + 1 :, :], target[idx + 1 :]

    ordinal_features = [
        x for x in range(len(feature_names)) if x not in list(category_map.keys())
    ]
    ordinal_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
        ]
    )

    categorical_features = list(category_map.keys())
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="median")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    preprocessor = ColumnTransformer(
        transformers=[
            ("num", ordinal_transformer, ordinal_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    clf = RandomForestClassifier(n_estimators=50)

    model_pipeline = Pipeline(
        steps=[
            ("preprocess", preprocessor),
            ("classifier", clf),
        ]
    )

    model_pipeline.fit(X_train, Y_train)

    explainer = AnchorTabular(
        model_pipeline.predict, feature_names, categorical_names=category_map, seed=1
    )

    explainer.fit(X_train, disc_perc=[25, 50, 75])

    if dirname is not None:
        explainer.save(dirname)
    return explainer
//  [python]
def fit(self, X):
        """A reference implementation of a fitting function for a transformer.
        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The training input samples.

        Returns
        -------
        self : object
            Returns self.
        """

        numeric_imputation = Imputer(strategy='median')

        X_df = pd.DataFrame(X)
        X_df = X_df.replace([np.inf, -np.inf], np.nan)

        numeric_imputation.fit(X_df.values)

        self.imputation = numeric_imputation

        return self
//  [python]
def fit(self, X):
        """A reference implementation of a fitting function for a transformer.
        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            The training input samples.

        Returns
        -------
        self : object
            Returns self.
        """

        numeric_imputation = SimpleImputer(strategy='median')

        X_df = pd.DataFrame(X)
        X_df = X_df.replace([np.inf, -np.inf], np.nan)

        numeric_imputation.fit(X_df.values)

        self.imputation = numeric_imputation

        return self
//  [python]
def _fit_lgb(self, X: pd.DataFrame, col: str):
        col_info = self.col_dict[col]
        null_mask = col_info['null_mask']
        _categorical_columns = self.categorical_columns.copy()
        if col in _categorical_columns:
            _categorical_columns.remove(col)
        if col_info['params'] is None:  # Single class
            model = Imputer()
            model.fit(X[col])
        else:
            params = col_info['params']
            params['verbose'] = -1
            x_train = X.loc[~null_mask].drop(col, axis=1)
            y_train = X.loc[~null_mask, col].copy()
            dtrain = lgb.Dataset(data=x_train, label=y_train)
            if self.fit_method == 'fit':
                model = lgb.train(
                    params, dtrain, valid_sets=[dtrain],
                    categorical_feature=_categorical_columns, **self.fit_params)
            elif self.fit_method == 'cv':
                res = lgb.cv(
                    params, dtrain, return_cvbooster=True,
                    stratified=False if col_info['params']['objective'] == 'regression' else True,
                    categorical_feature=_categorical_columns,
                    **self.fit_params)
                model = res['cvbooster']
        return model
//  [python]
def _fit_lgb(self, X: pd.DataFrame, col: str):
        col_info = self.col_dict[col]
        null_mask = col_info['null_mask']
        _categorical_columns = self.categorical_columns.copy()
        if col in _categorical_columns:
            _categorical_columns.remove(col)
        if col_info['params'] is None:  # Single class
            model = SimpleImputer()
            model.fit(X[col])
        else:
            params = col_info['params']
            params['verbose'] = -1
            x_train = X.loc[~null_mask].drop(col, axis=1)
            y_train = X.loc[~null_mask, col].copy()
            dtrain = lgb.Dataset(data=x_train, label=y_train)
            if self.fit_method == 'fit':
                model = lgb.train(
                    params, dtrain, valid_sets=[dtrain],
                    categorical_feature=_categorical_columns, **self.fit_params)
            elif self.fit_method == 'cv':
                res = lgb.cv(
                    params, dtrain, return_cvbooster=True,
                    stratified=False if col_info['params']['objective'] == 'regression' else True,
                    categorical_feature=_categorical_columns,
                    **self.fit_params)
                model = res['cvbooster']
        return model
//  [python]
set_config(display="diagram")

num_proc = make_pipeline(Imputer(strategy="median"), StandardScaler())
//  [python]
set_config(display="diagram")

num_proc = make_pipeline(SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def _benchmark(trial):
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.svm import LinearSVC
    from sklearn.calibration import CalibratedClassifierCV
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import roc_auc_score
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
    from sklearn.compose import ColumnTransformer
    from sklearn.impute import SimpleImputer

    if trial.task.problem == "binary" and trial.task.origin == "openml":
        X, y, meta = trial.task.data(["X", "y", "meta"])

        # Holdout split
        X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3)

        # Build preprocessor
        is_cat = meta["categorical_mask"]
        cat_cols = [idx for idx in range(X.shape[1]) if is_cat[idx]]
        num_cols = [idx for idx in range(X.shape[1]) if not is_cat[idx]]
        cat_ohe_step = ("ohe", OneHotEncoder(sparse=True, handle_unknown="ignore"))
        cat_pipe = Pipeline([cat_ohe_step])
        num_pipe = Pipeline([("identity", FunctionTransformer())])
        transformers = [("cat", cat_pipe, cat_cols), ("num", num_pipe, num_cols)]
        ct = Pipeline(
            [
                ("ct", ColumnTransformer(transformers=transformers)),
                (
                    "missing",
                    SimpleImputer(add_indicator=True, strategy="most_frequent"),
                ),
            ]
        )
        # Connect preprocessor with target learner
        if trial.method.name == "svm":
            clf = Pipeline([("ct", ct), ("est", CalibratedClassifierCV(LinearSVC()))])
        else:
            clf = Pipeline([("ct", ct), ("est", RandomForestClassifier())])

        # Train
        clf.fit(X_tr, y_tr)

        # Predict
        predictions = clf.predict_proba(X_te)[:, 1]

        # Score
        auc = roc_auc_score(y_te, predictions)
        trial.log("auc", auc)
//  [python]
def fit_imputer_and_scaler(self):
        X = self.X
        imp = Imputer()
        imp.fit(X)
        X = imp.transform(X)
        sc = StandardScaler()
        sc.fit(X)

        fimp = self.inputs["path_to_imputer"]
        fsc = self.inputs["path_to_scaler"]

        joblib.dump(imp, fimp)
        joblib.dump(sc, fsc)
        print("done")
//  [python]
def fit_imputer_and_scaler(self):
        X = self.X
        imp = SimpleImputer()
        imp.fit(X)
        X = imp.transform(X)
        sc = StandardScaler()
        sc.fit(X)

        fimp = self.inputs["path_to_imputer"]
        fsc = self.inputs["path_to_scaler"]

        joblib.dump(imp, fimp)
        joblib.dump(sc, fsc)
        print("done")
//  [python]
name_maps = {}
for feature_name in synthetic_data.columns:
    rred_name = re.sub("\W", ".", feature_name)
    if rred_name in train_data.columns:
        name_maps[rred_name] = feature_name
train_data = encoder(train_data.rename(name_maps, axis="columns"))
test_data = encoder(test_data.rename(name_maps, axis="columns"))
imp = Imputer(missing_values=np.nan, strategy='mean')
//  [python]
name_maps = {}
for feature_name in synthetic_data.columns:
    rred_name = re.sub("\W", ".", feature_name)
    if rred_name in train_data.columns:
        name_maps[rred_name] = feature_name
train_data = encoder(train_data.rename(name_maps, axis="columns"))
test_data = encoder(test_data.rename(name_maps, axis="columns"))
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
//  [python]
def preprocess_mlp(dataset, log):
    '''
    For MLP:
    - For numerical features, normalize them after null imputation. 
    - For categorical features, use one-hot encoding after null imputation. 
    '''
    cat_columns, num_columns = [], []
    shift_amount = 0
    for i, f in enumerate(dataset.features):
        if f.is_target:
            shift_amount += 1
            continue
        elif f.is_categorical():
            cat_columns.append(i - shift_amount)
        else:
            num_columns.append(i - shift_amount)

    cat_pipeline = Pipeline([('imputer', Imputer(strategy='most_frequent')),
                             ('onehot_encoder', OneHotEncoder()),
                             ])
    
    num_pipeline = Pipeline([('imputer', Imputer(strategy='mean')),
                             ('standard_scaler', StandardScaler()),
                             ])
    
    data_pipeline = ColumnTransformer([
        ('categorical', cat_pipeline, cat_columns),
        ('numerical', num_pipeline, num_columns),
    ])

    data_pipeline.fit(np.concatenate([dataset.train.X, dataset.test.X], axis=0))
    
    X_train = data_pipeline.transform(dataset.train.X)
    X_test = data_pipeline.transform(dataset.test.X)  
    
    return X_train, X_test
//  [python]
def preprocess_mlp(dataset, log):
    '''
    For MLP:
    - For numerical features, normalize them after null imputation. 
    - For categorical features, use one-hot encoding after null imputation. 
    '''
    cat_columns, num_columns = [], []
    shift_amount = 0
    for i, f in enumerate(dataset.features):
        if f.is_target:
            shift_amount += 1
            continue
        elif f.is_categorical():
            cat_columns.append(i - shift_amount)
        else:
            num_columns.append(i - shift_amount)

    cat_pipeline = Pipeline([('imputer', SimpleImputer(strategy='most_frequent')),
                             ('onehot_encoder', OneHotEncoder()),
                             ])
    
    num_pipeline = Pipeline([('imputer', SimpleImputer(strategy='mean')),
                             ('standard_scaler', StandardScaler()),
                             ])
    
    data_pipeline = ColumnTransformer([
        ('categorical', cat_pipeline, cat_columns),
        ('numerical', num_pipeline, num_columns),
    ])

    data_pipeline.fit(np.concatenate([dataset.train.X, dataset.test.X], axis=0))
    
    X_train = data_pipeline.transform(dataset.train.X)
    X_test = data_pipeline.transform(dataset.test.X)  
    
    return X_train, X_test