jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def get_impute_mean(X_missing, y_missing):
    imputer = Imputer(missing_values=np.nan, strategy="mean", add_indicator=True)
    mean_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing)
    return mean_impute_scores.mean(), mean_impute_scores.std()
//  [python]
def get_impute_mean(X_missing, y_missing):
    imputer = SimpleImputer(missing_values=np.nan, strategy="mean", add_indicator=True)
    mean_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing)
    return mean_impute_scores.mean(), mean_impute_scores.std()
//  [python]
def fill_missing_values(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()
    parameters = {}
    for col in df.columns[df.isnull().any()]:
        if (
            df[col].dtype == "float64"
            or df[col].dtype == "int64"
            or df[col].dtype == "int32"
        ):
            strategy = "mean"
        else:
            strategy = "most_frequent"
        missing_values = df[col][df[col].isnull()].values[0]
        parameters[col] = {"missing_values": missing_values, "strategy": strategy}

    for col, param in parameters.items():
        missing_values = param["missing_values"]
        strategy = param["strategy"]
        imp = Imputer(missing_values=missing_values, strategy=strategy)
        df[col] = imp.fit_transform(df[[col]])

    return df
//  [python]
def fill_missing_values(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()
    parameters = {}
    for col in df.columns[df.isnull().any()]:
        if (
            df[col].dtype == "float64"
            or df[col].dtype == "int64"
            or df[col].dtype == "int32"
        ):
            strategy = "mean"
        else:
            strategy = "most_frequent"
        missing_values = df[col][df[col].isnull()].values[0]
        parameters[col] = {"missing_values": missing_values, "strategy": strategy}

    for col, param in parameters.items():
        missing_values = param["missing_values"]
        strategy = param["strategy"]
        imp = SimpleImputer(missing_values=missing_values, strategy=strategy)
        df[col] = imp.fit_transform(df[[col]])

    return df
//  [python]
def __init__(self, pad_method="knn", use_conf=True) -> None:
        """
        pad_method (str): Specifies the method used to pad the missing values.
                        The supported methods are "knn" and "simple".
        use_conf (bool): Indicates whether confidence scores.
        """
        self.use_conf = use_conf
        if pad_method.lower() == "knn":
            self.imputer = KNNImputer(missing_values=0.0, n_neighbors=4, weights="distance", add_indicator=False)
        elif pad_method.lower()  == "simple":
            self.imputer = Imputer(missing_values=0.0, strategy='mean')
        else:
            raise ValueError(f"Error value for padding method: {pad_method}")
//  [python]
def __init__(self, pad_method="knn", use_conf=True) -> None:
        """
        pad_method (str): Specifies the method used to pad the missing values.
                        The supported methods are "knn" and "simple".
        use_conf (bool): Indicates whether confidence scores.
        """
        self.use_conf = use_conf
        if pad_method.lower() == "knn":
            self.imputer = KNNImputer(missing_values=0.0, n_neighbors=4, weights="distance", add_indicator=False)
        elif pad_method.lower()  == "simple":
            self.imputer = SimpleImputer(missing_values=0.0, strategy='mean',add_indicator=True)
        else:
            raise ValueError(f"Error value for padding method: {pad_method}")
//  [python]
plt.tight_layout()
plt.show()

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)


# instantiate a pca object and fit the model
pca = PCA(n_components=7)

pipe1 = make_pipeline(OutlierTrans(2),
  Imputer(strategy="median"), StandardScaler(), pca)
//  [python]
plt.tight_layout()
plt.show()

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)


# instantiate a pca object and fit the model
pca = PCA(n_components=7)

pipe1 = make_pipeline(OutlierTrans(2),
  SimpleImputer(strategy="median"), StandardScaler(), pca)
//  [python]
def _get_pipeline(self, max_depth: int, eta: float, min_child_weight: int,
                      colsample_bytree: float, colsample_bylevel: float, reg_lambda: int, reg_alpha: int,
                      n_estimators: int, subsample_per_it: float) \
            -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """
        objective = 'binary:logistic' if self.num_class <= 2 else 'multi:softmax'

        clf = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", Imputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", "passthrough", ~self.categorical_data)])),
            ('xgb',
             xgb.XGBClassifier(
                 max_depth=max_depth,
                 learning_rate=eta,
                 min_child_weight=min_child_weight,
                 colsample_bytree=colsample_bytree,
                 colsample_bylevel=colsample_bylevel,
                 reg_alpha=reg_alpha,
                 reg_lambda=reg_lambda,
                 n_estimators=n_estimators,
                 objective=objective,
                 n_jobs=self.n_threads,
                 random_state=self.rng.randint(1, 100000),
                 num_class=self.num_class,
                 subsample=subsample_per_it))
            ])
        return clf
//  [python]
def _get_pipeline(self, max_depth: int, eta: float, min_child_weight: int,
                      colsample_bytree: float, colsample_bylevel: float, reg_lambda: int, reg_alpha: int,
                      n_estimators: int, subsample_per_it: float) \
            -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """
        objective = 'binary:logistic' if self.num_class <= 2 else 'multi:softmax'

        clf = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", SimpleImputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", "passthrough", ~self.categorical_data)])),
            ('xgb',
             xgb.XGBClassifier(
                 max_depth=max_depth,
                 learning_rate=eta,
                 min_child_weight=min_child_weight,
                 colsample_bytree=colsample_bytree,
                 colsample_bylevel=colsample_bylevel,
                 reg_alpha=reg_alpha,
                 reg_lambda=reg_lambda,
                 n_estimators=n_estimators,
                 objective=objective,
                 n_jobs=self.n_threads,
                 random_state=self.rng.randint(1, 100000),
                 num_class=self.num_class,
                 subsample=subsample_per_it))
            ])
        return clf
//  [python]
def _get_pipeline(self, n_estimators: int, booster: str, reg_lambda: int, reg_alpha: int, eta: float,
                      min_child_weight: int = None, max_depth: int = None, colsample_bytree: float = None,
                      colsample_bylevel: float = None, subsample_per_it: float = None) \
            -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """
        objective = 'binary:logistic' if self.num_class <= 2 else 'multi:softmax'

        configuration = dict(booster=booster,
                             max_depth=max_depth,
                             learning_rate=eta,
                             min_child_weight=min_child_weight,
                             colsample_bytree=colsample_bytree,
                             colsample_bylevel=colsample_bylevel,
                             reg_alpha=reg_alpha,
                             reg_lambda=reg_lambda,
                             n_estimators=n_estimators,
                             objective=objective,
                             n_jobs=self.n_threads,
                             random_state=self.rng.randint(1, 100000),
                             num_class=self.num_class,
                             subsample=subsample_per_it)

        configuration = {k: v for k, v in configuration.items() if v is not None}

        clf = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", Imputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", "passthrough", ~self.categorical_data)])),
            ('xgb',
             xgb.XGBClassifier(**configuration))
        ])
        return clf
//  [python]
def _get_pipeline(self, n_estimators: int, booster: str, reg_lambda: int, reg_alpha: int, eta: float,
                      min_child_weight: int = None, max_depth: int = None, colsample_bytree: float = None,
                      colsample_bylevel: float = None, subsample_per_it: float = None) \
            -> pipeline.Pipeline:
        """ Create the scikit-learn (training-)pipeline """
        objective = 'binary:logistic' if self.num_class <= 2 else 'multi:softmax'

        configuration = dict(booster=booster,
                             max_depth=max_depth,
                             learning_rate=eta,
                             min_child_weight=min_child_weight,
                             colsample_bytree=colsample_bytree,
                             colsample_bylevel=colsample_bylevel,
                             reg_alpha=reg_alpha,
                             reg_lambda=reg_lambda,
                             n_estimators=n_estimators,
                             objective=objective,
                             n_jobs=self.n_threads,
                             random_state=self.rng.randint(1, 100000),
                             num_class=self.num_class,
                             subsample=subsample_per_it)

        configuration = {k: v for k, v in configuration.items() if v is not None}

        clf = pipeline.Pipeline([
            ('preprocess_impute',
             ColumnTransformer([
                 ("categorical", "passthrough", self.categorical_data),
                 ("continuous", SimpleImputer(strategy="mean"), ~self.categorical_data)])),
            ('preprocess_one_hot',
             ColumnTransformer([
                 ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data),
                 ("continuous", "passthrough", ~self.categorical_data)])),
            ('xgb',
             xgb.XGBClassifier(**configuration))
        ])
        return clf
//  [python]
def _create_sklearn_object(self) -> impute.SimpleImputer:
        """
        Get an equivalent sklearn SimpleImputer.

        Returns:
            Sklearn SimpleImputer.
        """
        sklearn_args = self.get_sklearn_args(
            default_sklearn_obj=impute.SimpleImputer(),
            sklearn_initial_keywords=_SKLEARN_INITIAL_KEYWORDS,
            sklearn_unused_keywords=_SKLEARN_UNUSED_KEYWORDS,
            snowml_only_keywords=_SNOWML_ONLY_KEYWORDS,
            sklearn_added_keyword_to_version_dict=_SKLEARN_ADDED_KEYWORD_TO_VERSION_DICT,
        )

        simple_imputer = impute.SimpleImputer(**sklearn_args)
        if self._is_fitted:
            simple_imputer.statistics_ = np.array(list(self.statistics_.values()))
            simple_imputer.n_features_in_ = self.n_features_in_
            simple_imputer.feature_names_in_ = self.feature_names_in_
            simple_imputer._fit_dtype = self._sklearn_fit_dtype

        return simple_imputer
//  [python]
def prepare_data_for_corr(data):

    has_nan = data.isna().any().any()

    if has_nan:
    
        Numerical_col = seperate_cat_num_cols(data)[0]
        Categorical_col = seperate_cat_num_cols(data)[1]

        cat_col_index,num_col_index = [],[]            
        for i in Categorical_col:
            cat_col_index.append(data.columns.tolist().index(i))
        for i in Numerical_col:
            num_col_index.append(data.columns.tolist().index(i))

        #Factorize categorical feature
        for i in range(len(Categorical_col)):
            data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]

        data = data.replace(-1, np.nan)

        num_data_for_corr = data.copy()
        imputer1 = Imputer(missing_values=np.nan, strategy='mean')
        imputer1 = imputer1.fit(num_data_for_corr.values[:,num_col_index])
        
        num_data_for_corr = imputer1.transform(num_data_for_corr.values[:,num_col_index])
        num_data_for_corr = pd.DataFrame(num_data_for_corr,columns = Numerical_col)
        
        if len(cat_col_index) > 0:
            Cat_data_for_corr = data.copy()
            imputer2 = Imputer(missing_values=np.nan, strategy='most_frequent')
            imputer2 = imputer2.fit(Cat_data_for_corr.values[:,cat_col_index])
            Cat_data_for_corr = imputer2.transform(Cat_data_for_corr.values[:,cat_col_index])
            Cat_data_for_corr = pd.DataFrame(Cat_data_for_corr,columns=Categorical_col)
            data_for_corr = pd.concat([num_data_for_corr, Cat_data_for_corr], axis=1)
        
        else:
            data_for_corr = num_data_for_corr
            
        data_for_corr = data_for_corr.astype(float)
    else:
        data_for_corr = data

    return data_for_corr
//  [python]
def prepare_data_for_corr(data):

    has_nan = data.isna().any().any()

    if has_nan:
    
        Numerical_col = seperate_cat_num_cols(data)[0]
        Categorical_col = seperate_cat_num_cols(data)[1]

        cat_col_index,num_col_index = [],[]            
        for i in Categorical_col:
            cat_col_index.append(data.columns.tolist().index(i))
        for i in Numerical_col:
            num_col_index.append(data.columns.tolist().index(i))

        #Factorize categorical feature
        for i in range(len(Categorical_col)):
            data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]

        data = data.replace(-1, np.nan)

        num_data_for_corr = data.copy()
        imputer1 = SimpleImputer(missing_values=np.nan, strategy='mean')
        imputer1 = imputer1.fit(num_data_for_corr.values[:,num_col_index])
        
        num_data_for_corr = imputer1.transform(num_data_for_corr.values[:,num_col_index])
        num_data_for_corr = pd.DataFrame(num_data_for_corr,columns = Numerical_col)
        
        if len(cat_col_index) > 0:
            Cat_data_for_corr = data.copy()
            imputer2 = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
            imputer2 = imputer2.fit(Cat_data_for_corr.values[:,cat_col_index])
            Cat_data_for_corr = imputer2.transform(Cat_data_for_corr.values[:,cat_col_index])
            Cat_data_for_corr = pd.DataFrame(Cat_data_for_corr,columns=Categorical_col)
            data_for_corr = pd.concat([num_data_for_corr, Cat_data_for_corr], axis=1)
        
        else:
            data_for_corr = num_data_for_corr
            
        data_for_corr = data_for_corr.astype(float)
    else:
        data_for_corr = data

    return data_for_corr
//  [python]
def _fill_nan(self):
        """Missing value imputation.

        For fields with type :obj:`~recbole.utils.enum_type.FeatureType.TOKEN`, missing value will be filled by
        ``[PAD]``, which indexed as 0.

        For fields with type :obj:`~recbole.utils.enum_type.FeatureType.FLOAT`, missing value will be filled by
        the average of original data.

        For sequence features, missing value will be filled by ``[0]``. 
        """
        self.logger.debug('Filling nan')

        if not self._fill_nan_flag:
            return

        from sklearn.preprocessing import Imputer
        most_freq = Imputer(missing_values=np.nan, strategy='most_frequent', copy=False)
        aveg = Imputer(missing_values=np.nan, strategy='mean', copy=False)

        for feat in self.feat_list:
            for field in feat:
                ftype = self.field2type[field]
                if ftype == FeatureType.TOKEN:
                    feat[field] = most_freq.fit_transform(feat[field].values.reshape(-1, 1))
                elif ftype == FeatureType.FLOAT:
                    feat[field] = aveg.fit_transform(feat[field].values.reshape(-1, 1))
                elif ftype.value.endswith('seq'):
                    feat[field] = feat[field].apply(lambda x: [0]
                                                    if (not isinstance(x, np.ndarray) and (not isinstance(x, list)))
                                                    else x)
//  [python]
def _fill_nan(self):
        """Missing value imputation.

        For fields with type :obj:`~recbole.utils.enum_type.FeatureType.TOKEN`, missing value will be filled by
        ``[PAD]``, which indexed as 0.

        For fields with type :obj:`~recbole.utils.enum_type.FeatureType.FLOAT`, missing value will be filled by
        the average of original data.

        For sequence features, missing value will be filled by ``[0]``. 
        """
        self.logger.debug('Filling nan')

        if not self._fill_nan_flag:
            return

        most_freq = SimpleImputer(missing_values=np.nan, strategy='most_frequent', copy=False)
        aveg = SimpleImputer(missing_values=np.nan, strategy='mean', copy=False)

        for feat in self.feat_list:
            for field in feat:
                ftype = self.field2type[field]
                if ftype == FeatureType.TOKEN:
                    feat[field] = most_freq.fit_transform(feat[field].values.reshape(-1, 1))
                elif ftype == FeatureType.FLOAT:
                    feat[field] = aveg.fit_transform(feat[field].values.reshape(-1, 1))
                elif ftype.value.endswith('seq'):
                    feat[field] = feat[field].apply(lambda x: [0]
                                                    if (not isinstance(x, np.ndarray) and (not isinstance(x, list)))
                                                    else x)
//  [python]
def fill_nan_categoric_cols(data,classifiation_estimator,S):
    
    Numerical_col = seperate_cat_num_cols(data)[0]
    Categorical_col = seperate_cat_num_cols(data)[1]
    correlated_features = S
    
    for i in range(len(Categorical_col)):
        data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]
        
    data = data.replace(-1, np.nan)
    data_org = data.copy()
   
    for i in Categorical_col:
        columns = []
        m = correlated_features[i]
        #m.remove(i)
       
        cat_data = data[m].copy()
        label = data[i]
        
        nan=[]
        fill = [] 
        for k in range(data.shape[0]):
        
            if data[i].isnull()[k] ==True:
                nan.append(k)
            else:
                fill.append(k)
        
        imputer = Imputer(missing_values=np.nan, strategy='most_frequent')
        imputer = imputer.fit(cat_data.values)
        transform = imputer.transform(cat_data.values)
        cat_data = pd.DataFrame(transform)
        cat_data[i] = label
        
        X = cat_data.values[fill,:-1]
        Y = cat_data.values[fill,-1]

        X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.2, random_state=0)
        classify = classifiation_estimator
        classify.fit(X_train, y_train)  
        predicted = classify.predict(cat_data.values[nan,:-1])
        
        
      #Fill nan for i column with predicted valus
        
        k = 0
        for t in nan:
            data[i][t] = predicted[k]
            k = k+1

    return data
//  [python]
def fill_nan_categoric_cols(data,classifiation_estimator,S):
    
    Numerical_col = seperate_cat_num_cols(data)[0]
    Categorical_col = seperate_cat_num_cols(data)[1]
    correlated_features = S
    
    for i in range(len(Categorical_col)):
        data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]
        
    data = data.replace(-1, np.nan)
    data_org = data.copy()
   
    for i in Categorical_col:
        columns = []
        m = correlated_features[i]
        #m.remove(i)
       
        cat_data = data[m].copy()
        label = data[i]
        
        nan=[]
        fill = [] 
        for k in range(data.shape[0]):
        
            if data[i].isnull()[k] ==True:
                nan.append(k)
            else:
                fill.append(k)
        
        imputer = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
        imputer = imputer.fit(cat_data.values)
        transform = imputer.transform(cat_data.values)
        cat_data = pd.DataFrame(transform)
        cat_data[i] = label
        
        X = cat_data.values[fill,:-1]
        Y = cat_data.values[fill,-1]

        X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.2, random_state=0)
        classify = classifiation_estimator
        classify.fit(X_train, y_train)  
        predicted = classify.predict(cat_data.values[nan,:-1])
        
        
      #Fill nan for i column with predicted valus
        
        k = 0
        for t in nan:
            data[i][t] = predicted[k]
            k = k+1

    return data
//  [python]
def fill_nan_numeric_cols(data,regression_estimator,S):
    
    data_org = data.copy()
    Numerical_col = seperate_cat_num_cols(data)[0]
    Categorical_col = seperate_cat_num_cols(data)[1]
    correlated_features = S
    
    for i in range(len(Categorical_col)):
        data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]

    data = data.replace(-1, np.nan)
    
    for i in Numerical_col:
        
        columns = []
        m = correlated_features[i]
        #m.remove(i)
        
        num_data = data[m].copy()
        
        label = data[i]
        """need_scale = []  #If you need to scale the numerical columns you can uncomment this and work a little on it. 
        for j in correlated_features[i]:
            if j in Numerical_col:
                need_scale.append(j)"""
        
        #Scaler = scaler    
        #num_data[need_scale] = Scaler.fit_transform(num_data[need_scale])
        

        #print(data)
        nan=[]
        fill = [] 
        for k in range(data.shape[0]):
            
            if data[i].isnull()[k] ==True: 
                nan.append(k)
            else:
                fill.append(k)
        
        #Fill nan in num_data with SimpleImputer
        
        imputer = Imputer(missing_values=np.nan, strategy='most_frequent')
        imputer = imputer.fit(num_data.values)
        transform = imputer.transform(num_data.values)
        num_data = pd.DataFrame(transform)
        num_data[i] = label
        
        #Train regression model
        
        X = num_data.values[fill,:-1]
        Y = num_data.values[fill,-1]
    
        X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.2, random_state=0)
        reg = regression_estimator
        reg.fit(X_train, y_train)  
        predicted = reg.predict(num_data.values[nan,:-1])
        
        k = 0
        for t in nan:
            data[i][t] = predicted[k]
            k = k+1

    data.drop(Categorical_col,axis=1,inplace=True)
    
    for i in Categorical_col:
        data[i] = data_org[i]
        
    return data
//  [python]
def fill_nan_numeric_cols(data,regression_estimator,S):
    
    data_org = data.copy()
    Numerical_col = seperate_cat_num_cols(data)[0]
    Categorical_col = seperate_cat_num_cols(data)[1]
    correlated_features = S
    
    for i in range(len(Categorical_col)):
        data[Categorical_col[i]] = pd.factorize(data[Categorical_col[i]])[0]

    data = data.replace(-1, np.nan)
    
    for i in Numerical_col:
        
        columns = []
        m = correlated_features[i]
        #m.remove(i)
        
        num_data = data[m].copy()
        
        label = data[i]
        """need_scale = []  #If you need to scale the numerical columns you can uncomment this and work a little on it. 
        for j in correlated_features[i]:
            if j in Numerical_col:
                need_scale.append(j)"""
        
        #Scaler = scaler    
        #num_data[need_scale] = Scaler.fit_transform(num_data[need_scale])
        

        #print(data)
        nan=[]
        fill = [] 
        for k in range(data.shape[0]):
            
            if data[i].isnull()[k] ==True: 
                nan.append(k)
            else:
                fill.append(k)
        
        #Fill nan in num_data with SimpleImputer
        
        imputer = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
        imputer = imputer.fit(num_data.values)
        transform = imputer.transform(num_data.values)
        num_data = pd.DataFrame(transform)
        num_data[i] = label
        
        #Train regression model
        
        X = num_data.values[fill,:-1]
        Y = num_data.values[fill,-1]
    
        X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.2, random_state=0)
        reg = regression_estimator
        reg.fit(X_train, y_train)  
        predicted = reg.predict(num_data.values[nan,:-1])
        
        k = 0
        for t in nan:
            data[i][t] = predicted[k]
            k = k+1

    data.drop(Categorical_col,axis=1,inplace=True)
    
    for i in Categorical_col:
        data[i] = data_org[i]
        
    return data
//  [python]
def get_data_transformation_object(self):
        '''
        This function is responsible for data transformation
        '''
        try:
            
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']

            # Numerical Pipeline
            num_pipeline = Pipeline(
                steps = [
                ('imputer',Imputer(strategy='median')),
                ('scaler',StandardScaler())                
                ]
            )

            # Categorical Pipeline
            cat_pipeline = Pipeline(
                steps=[
                ('imputer',Imputer(strategy='most_frequent')),
                ('ordinal_encoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]
            )

            logging.info(f'Categorical Columns : {categorical_cols}')
            logging.info(f'Numerical Columns   : {numerical_cols}')

            preprocessor = ColumnTransformer(
                [
                ('num_pipeline',num_pipeline,numerical_cols),
                ('cat_pipeline',cat_pipeline,categorical_cols)
                ]
            )

            return preprocessor
        
        except Exception as e:
            logging.info('Exception occured in Data Transformation Phase')
            raise CustomException(e,sys)
//  [python]
def get_data_transformation_object(self):
        '''
        This function is responsible for data transformation
        '''
        try:
            
            # Define which columns should be ordinal-encoded and which should be scaled
            categorical_cols = ['cut', 'color','clarity']
            numerical_cols = ['carat', 'depth','table', 'x', 'y', 'z']
            
            # Define the custom ranking for each ordinal variable
            cut_categories = ['Fair', 'Good', 'Very Good','Premium','Ideal']
            color_categories = ['D', 'E', 'F', 'G', 'H', 'I', 'J']
            clarity_categories = ['I1','SI2','SI1','VS2','VS1','VVS2','VVS1','IF']

            # Numerical Pipeline
            num_pipeline = Pipeline(
                steps = [
                ('imputer',SimpleImputer(strategy='median')),
                ('scaler',StandardScaler())                
                ]
            )

            # Categorical Pipeline
            cat_pipeline = Pipeline(
                steps=[
                ('imputer',SimpleImputer(strategy='most_frequent')),
                ('ordinal_encoder',OrdinalEncoder(categories=[cut_categories,color_categories,clarity_categories])),
                ('scaler',StandardScaler())
                ]
            )

            logging.info(f'Categorical Columns : {categorical_cols}')
            logging.info(f'Numerical Columns   : {numerical_cols}')

            preprocessor = ColumnTransformer(
                [
                ('num_pipeline',num_pipeline,numerical_cols),
                ('cat_pipeline',cat_pipeline,categorical_cols)
                ]
            )

            return preprocessor
        
        except Exception as e:
            logging.info('Exception occured in Data Transformation Phase')
            raise CustomException(e,sys)
//  [python]
X = dat.data
Y = dat.target
print("Examples = ",X.shape ," Labels = ", Y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 
                                                    0.3, random_state=100)
model_pipeline = Pipeline(steps=[
  ("feature_union", FeatureUnion([
    ('missing_values',Imputer(missing_values='NaN', strategy='mean')),
    ('scale', StandardScaler()),
    ("reduce_dim", PCA(n_components=10)),
  ])),
  ('clf', SVC(kernel='rbf', gamma= 0.001, C=5)) ])
//  [python]
X = dat.data
Y = dat.target
print("Examples = ",X.shape ," Labels = ", Y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 
                                                    0.3, random_state=100)
model_pipeline = Pipeline(steps=[
  ("feature_union", FeatureUnion([
    ('missing_values',SimpleImputer(missing_values=np.nan, strategy='mean')),
    ('scale', StandardScaler()),
    ("reduce_dim", PCA(n_components=10)),
  ])),
  ('clf', SVC(kernel='rbf', gamma= 0.001, C=5)) ])
//  [python]
def _preprocess_train(self, X, feature_types, vect_max_features):
        transformer_list = []
        if feature_types.get("language", None):
            pipeline = Pipeline(
                steps=[
                    ("preparator", NlpDataPreprocessor(nlp_cols=feature_types["language"])),
                    (
                        "vectorizer",
                        TfidfVectorizer(
                            ngram_range=self.params["proc.ngram_range"], sublinear_tf=True, max_features=vect_max_features, tokenizer=self._tokenize
                        ),
                    ),
                ]
            )
            transformer_list.append(("vect", pipeline, feature_types["language"]))
        if feature_types.get("onehot", None):
            pipeline = Pipeline(
                steps=[
                    ("generator", OheFeaturesGenerator()),
                ]
            )
            transformer_list.append(("cats", pipeline, feature_types["onehot"]))
        if feature_types.get("continuous", None):
            pipeline = Pipeline(steps=[("imputer", Imputer(strategy=self.params["proc.impute_strategy"])), ("scaler", StandardScaler())])
            transformer_list.append(("cont", pipeline, feature_types["continuous"]))
        if feature_types.get("bool", None):
            pipeline = Pipeline(steps=[("scaler", StandardScaler())])
            transformer_list.append(("bool", pipeline, feature_types["bool"]))
        if feature_types.get("skewed", None):
            pipeline = Pipeline(
                steps=[
                    ("imputer", Imputer(strategy=self.params["proc.impute_strategy"])),
                    ("quantile", QuantileTransformer(output_distribution="normal")),  # Or output_distribution = 'uniform'
                ]
            )
            transformer_list.append(("skew", pipeline, feature_types["skewed"]))
        self._pipeline = ColumnTransformer(transformers=transformer_list)
        return self._pipeline.fit_transform(X)
//  [python]
def _preprocess_train(self, X, feature_types, vect_max_features):
        transformer_list = []
        if feature_types.get("language", None):
            pipeline = Pipeline(
                steps=[
                    ("preparator", NlpDataPreprocessor(nlp_cols=feature_types["language"])),
                    (
                        "vectorizer",
                        TfidfVectorizer(
                            ngram_range=self.params["proc.ngram_range"], sublinear_tf=True, max_features=vect_max_features, tokenizer=self._tokenize
                        ),
                    ),
                ]
            )
            transformer_list.append(("vect", pipeline, feature_types["language"]))
        if feature_types.get("onehot", None):
            pipeline = Pipeline(
                steps=[
                    ("generator", OheFeaturesGenerator()),
                ]
            )
            transformer_list.append(("cats", pipeline, feature_types["onehot"]))
        if feature_types.get("continuous", None):
            pipeline = Pipeline(steps=[("imputer", SimpleImputer(strategy=self.params["proc.impute_strategy"])), ("scaler", StandardScaler())])
            transformer_list.append(("cont", pipeline, feature_types["continuous"]))
        if feature_types.get("bool", None):
            pipeline = Pipeline(steps=[("scaler", StandardScaler())])
            transformer_list.append(("bool", pipeline, feature_types["bool"]))
        if feature_types.get("skewed", None):
            pipeline = Pipeline(
                steps=[
                    ("imputer", SimpleImputer(strategy=self.params["proc.impute_strategy"])),
                    ("quantile", QuantileTransformer(output_distribution="normal")),  # Or output_distribution = 'uniform'
                ]
            )
            transformer_list.append(("skew", pipeline, feature_types["skewed"]))
        self._pipeline = ColumnTransformer(transformers=transformer_list)
        return self._pipeline.fit_transform(X)
//  [python]
def mean_median_imputer(
    dataframe: pd.DataFrame,
    imputation_method: str,
) -> pd.DataFrame:
    dataframe = dataframe.replace("?", np.nan)
    if imputation_method not in ["median", "mean"]:
        raise ValueError("imputation_method takes only values 'median' or 'mean'")

    imputer = Imputer(missing_values=np.nan, strategy=imputation_method)

    imputer = imputer.fit(dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]])
    dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]] = imputer.transform(
        dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]]
    )
    return dataframe
//  [python]
def mean_median_imputer(
    dataframe: pd.DataFrame,
    imputation_method: str,
) -> pd.DataFrame:
    dataframe = dataframe.replace("?", np.nan)
    if imputation_method not in ["median", "mean"]:
        raise ValueError("imputation_method takes only values 'median' or 'mean'")

    imputer = SimpleImputer(missing_values=np.nan, strategy=imputation_method)

    imputer = imputer.fit(dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]])
    dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]] = imputer.transform(
        dataframe[dataframe.columns[~dataframe.columns.isin(NO_IMPUTATION_COLS)]]
    )
    return dataframe
//  [python]
def catboost_pipeline(X):
    catboost_preprocessing = ColumnTransformer(
        transformers=[
            ("num", "passthrough", DataSelector.NumSelector),
            (
                "cat",
                Imputer(strategy="constant", fill_value=""),
                DataSelector.CatSelector,
            ),
            # ("txt", Imputer(strategy="constant", fill_value=""), DataSelector.TxtSelector),
        ],
        remainder="drop",
    )

    return Pipeline(
        steps=[
            ("preprocessing", catboost_preprocessing),
            ("model", CatBoostClassifier_wrapper()),
        ]
    )
//  [python]
def catboost_pipeline(X):
    catboost_preprocessing = ColumnTransformer(
        transformers=[
            ("num", "passthrough", DataSelector.NumSelector),
            (
                "cat",
                SimpleImputer(strategy="constant", fill_value=""),
                DataSelector.CatSelector,
            ),
            # ("txt", SimpleImputer(strategy="constant", fill_value=""), DataSelector.TxtSelector),
        ],
        remainder="drop",
    )

    return Pipeline(
        steps=[
            ("preprocessing", catboost_preprocessing),
            ("model", CatBoostClassifier_wrapper()),
        ]
    )
//  [python]
def get_mandatory_preprocessing(X, y):
    
    # determine fixed pre-processing steps for imputation and binarization (naml does this automatically)
    types = [set([type(v) for v in r]) for r in X.T]
    numeric_features = [c for c, t in enumerate(types) if len(t) == 1 and list(t)[0] != str]
    numeric_transformer = Pipeline([("imputer", sklearn.preprocessing.Imputer(strategy="median"))])
    categorical_features = [i for i in range(X.shape[1]) if i not in numeric_features]
    missing_values_per_feature = np.sum(pd.isnull(X), axis=0)
    if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
        categorical_transformer = Pipeline([
            ("imputer", sklearn.preprocessing.Imputer(strategy="most_frequent")),
            ("binarizer", sklearn.preprocessing.OneHotEncoder(handle_unknown='ignore', sparse = True)),

        ])
        mandatory_pre_processing = [("impute_and_binarize", ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        ))]
    else:
        mandatory_pre_processing = []
    return mandatory_pre_processing
//  [python]
def get_mandatory_preprocessing(X, y):
    
    # determine fixed pre-processing steps for imputation and binarization (naml does this automatically)
    types = [set([type(v) for v in r]) for r in X.T]
    numeric_features = [c for c, t in enumerate(types) if len(t) == 1 and list(t)[0] != str]
    numeric_transformer = Pipeline([("imputer", sklearn.impute.SimpleImputer(strategy="median"))])
    categorical_features = [i for i in range(X.shape[1]) if i not in numeric_features]
    missing_values_per_feature = np.sum(pd.isnull(X), axis=0)
    if len(categorical_features) > 0 or sum(missing_values_per_feature) > 0:
        categorical_transformer = Pipeline([
            ("imputer", sklearn.impute.SimpleImputer(strategy="most_frequent")),
            ("binarizer", sklearn.preprocessing.OneHotEncoder(handle_unknown='ignore', sparse = True)),

        ])
        mandatory_pre_processing = [("impute_and_binarize", ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        ))]
    else:
        mandatory_pre_processing = []
    return mandatory_pre_processing
//  [python]
def _simple_imputation(self, numeric_imputation, categorical_imputation):
        """Perform simple imputation of missing values."""
        self.logger.info("Set up simple imputation.")

        # Numerical imputation
        num_dict = {"mode": "most_frequent", "mean": "mean", "median": "median"}
        if isinstance(numeric_imputation, str):
            if numeric_imputation.lower() == "drop":
                num_estimator = TransformerWrapper(
                    transformer=DropImputer(columns=self._fxs["Numeric"])
                )
            elif numeric_imputation.lower() == "knn":
                num_estimator = TransformerWrapper(
                    transformer=KNNImputer(),
                    include=self._fxs["Numeric"],
                )
            elif numeric_imputation.lower() in num_dict:
                num_estimator = TransformerWrapper(
                    Imputer(strategy=num_dict[numeric_imputation.lower()]),
                    include=self._fxs["Numeric"],
                )
            else:
                raise ValueError(
                    "Invalid value for the numeric_imputation parameter, got "
                    f"{numeric_imputation}. Choose from: drop, mean, median, mode, knn."
                )
        else:
            num_estimator = TransformerWrapper(
                Imputer(strategy="constant", fill_value=numeric_imputation),
                include=self._fxs["Numeric"],
            )

        if categorical_imputation.lower() == "drop":
            cat_estimator = TransformerWrapper(
                transformer=DropImputer(columns=self._fxs["Categorical"])
            )
        elif categorical_imputation.lower() == "mode":
            cat_estimator = TransformerWrapper(
                transformer=Imputer(strategy="most_frequent"),
                include=self._fxs["Categorical"],
            )
        else:
            cat_estimator = TransformerWrapper(
                Imputer(strategy="constant", fill_value=categorical_imputation),
                include=self._fxs["Categorical"],
            )

        self.pipeline.steps.extend(
            [
                ("numerical_imputer", num_estimator),
                ("categorical_imputer", cat_estimator),
            ],
        )
//  [python]
def _simple_imputation(self, numeric_imputation, categorical_imputation):
        """Perform simple imputation of missing values."""
        self.logger.info("Set up simple imputation.")

        # Numerical imputation
        num_dict = {"mode": "most_frequent", "mean": "mean", "median": "median"}
        if isinstance(numeric_imputation, str):
            if numeric_imputation.lower() == "drop":
                num_estimator = TransformerWrapper(
                    transformer=DropImputer(columns=self._fxs["Numeric"])
                )
            elif numeric_imputation.lower() == "knn":
                num_estimator = TransformerWrapper(
                    transformer=KNNImputer(),
                    include=self._fxs["Numeric"],
                )
            elif numeric_imputation.lower() in num_dict:
                num_estimator = TransformerWrapper(
                    SimpleImputer(strategy=num_dict[numeric_imputation.lower()]),
                    include=self._fxs["Numeric"],
                )
            else:
                raise ValueError(
                    "Invalid value for the numeric_imputation parameter, got "
                    f"{numeric_imputation}. Choose from: drop, mean, median, mode, knn."
                )
        else:
            num_estimator = TransformerWrapper(
                SimpleImputer(strategy="constant", fill_value=numeric_imputation),
                include=self._fxs["Numeric"],
            )

        if categorical_imputation.lower() == "drop":
            cat_estimator = TransformerWrapper(
                transformer=DropImputer(columns=self._fxs["Categorical"])
            )
        elif categorical_imputation.lower() == "mode":
            cat_estimator = TransformerWrapper(
                transformer=SimpleImputer(strategy="most_frequent"),
                include=self._fxs["Categorical"],
            )
        else:
            cat_estimator = TransformerWrapper(
                SimpleImputer(strategy="constant", fill_value=categorical_imputation),
                include=self._fxs["Categorical"],
            )

        self.pipeline.steps.extend(
            [
                ("numerical_imputer", num_estimator),
                ("categorical_imputer", cat_estimator),
            ],
        )
//  [python]
def __init__(self, f_df, discrete=False, y_one_hot=True, drop='first'):
        self.f_df = f_df
        self.discrete = discrete
        self.y_one_hot = y_one_hot
        self.label_enc = preprocessing.OneHotEncoder(categories='auto') if y_one_hot else preprocessing.LabelEncoder()
        self.feature_enc = preprocessing.OneHotEncoder(categories='auto', drop=drop)
        self.imp = Imputer(missing_values=np.nan, strategy='mean')
        self.X_fname = None
        self.y_fname = None
        self.discrete_flen = None
        self.continuous_flen = None
        self.mean = None
        self.std = None
//  [python]
def __init__(self, f_df, discrete=False, y_one_hot=True, drop='first'):
        self.f_df = f_df
        self.discrete = discrete
        self.y_one_hot = y_one_hot
        self.label_enc = preprocessing.OneHotEncoder(categories='auto') if y_one_hot else preprocessing.LabelEncoder()
        self.feature_enc = preprocessing.OneHotEncoder(categories='auto', drop=drop)
        self.imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        self.X_fname = None
        self.y_fname = None
        self.discrete_flen = None
        self.continuous_flen = None
        self.mean = None
        self.std = None
//  [python]
def fit(self, X, y=None): 

        # Create and fit simple imputer
        self.imputer = Imputer(strategy='median')
        self.imputer.fit(X[self.numerical_columns])
        
        # Create and fit Standard Scaler 
        self.scaler = StandardScaler()
        self.scaler.fit(X[self.numerical_columns]) 
        
        # Create and fit one hot encoder
        self.onehot = OneHotEncoder(handle_unknown='ignore')
        self.onehot.fit(X[self.categorical_columns])
        
        return self
//  [python]
def fit(self, X, y=None): 

        # Create and fit simple imputer
        self.imputer = SimpleImputer(strategy='median')
        self.imputer.fit(X[self.numerical_columns])
        
        # Create and fit Standard Scaler 
        self.scaler = StandardScaler()
        self.scaler.fit(X[self.numerical_columns]) 
        
        # Create and fit one hot encoder
        self.onehot = OneHotEncoder(handle_unknown='ignore')
        self.onehot.fit(X[self.categorical_columns])
        
        return self
//  [python]
def __init__(self, num_strategy="mean"):
        super(Preprocessor, self).__init__()
        self.num_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy=num_strategy)),
                ("scaler", MinMaxScaler()),
            ]
        )
        self.feature_enc = OneHotEncoder(sparse=False, handle_unknown="ignore")
        self.cat_imputer = Imputer(strategy="constant", fill_value="missing")
        self.label_enc = LabelEncoder()
//  [python]
def __init__(self, num_strategy="mean"):
        super(Preprocessor, self).__init__()
        self.num_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy=num_strategy)),
                ("scaler", MinMaxScaler()),
            ]
        )
        self.feature_enc = OneHotEncoder(sparse=False, handle_unknown="ignore")
        self.cat_imputer = SimpleImputer(strategy="constant", fill_value="missing")
        self.label_enc = LabelEncoder()
//  [python]
x = np.zeros((len(icu_id), 12 , 48))
x_lab = np.zeros((len(icu_id), len(lab_IDs) , 48))
x_impute = np.zeros((len(icu_id), 12, 48))
y = np.zeros((len(icu_id),))
imp_mean = Imputer(strategy="mean")
//  [python]
x = np.zeros((len(icu_id), 12 , 48))
x_lab = np.zeros((len(icu_id), len(lab_IDs) , 48))
x_impute = np.zeros((len(icu_id), 12, 48))
y = np.zeros((len(icu_id),))
imp_mean = SimpleImputer(strategy="mean")