Kaggle competition: Categorical Feature Encoding Challenge
def get_dataloaders_for_inference(
 file_list: tp.List[tp.List], batch_size=64, 
):

 dataset = LabeledImageDataset(
 file_list,
transform_list=[
["Normalize", {
"always_apply": True, "max_pixel_value": 255.0,
"mean": ["0.4887381077884414"], "std": ["0.23064819430546407"]}],
["ToTensorV2", {"always_apply": True}],
])
 loader = data.DataLoader(
 dataset,
batch_size=batch_size, shuffle=False,
num_workers=2, pin_memory=True,
drop_last=False)
 return loader<categorify>
with timer("Load"):
nrow = None
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv", index_col = 'id', nrows = nrow)
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv", index_col = 'id')
 submission_df = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 [print(x.shape)for x in [train, test, submission_df]]
traindex = train.index
testdex = test.index
y = train.target.copy()
df = pd.concat([train.drop('target',axis = 1), test], axis = 0)
 del train, test, submission_df
class ImageTransformBase:

 def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):

 augmentations_list = [
self._get_augmentation(aug_name )(**params)
 for aug_name, params in data_augmentations]
self.data_aug = albumentations.Compose(augmentations_list)
 def __call__(self, pair: tp.Tuple[np.ndarray])-> tp.Tuple[np.ndarray]:

 raise NotImplementedError
def _get_augmentation(self, aug_name: str)-> tp.Tuple[ImageOnlyTransform, DualTransform]:

 if hasattr(albumentations, aug_name):
return getattr(albumentations, aug_name)
 else:
return eval(aug_name)
 class ImageTransformForCls(ImageTransformBase):

 def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):

 super(ImageTransformForCls, self ).__init__(data_augmentations)
 def __call__(self, in_arrs: tp.Tuple[np.ndarray])-> tp.Tuple[np.ndarray]:

 img, label = in_arrs
augmented = self.data_aug(image=img)
 img = augmented["image"]
return img, label<load_pretrained>
with timer("Categorical Processing"):
categorical = df.columns
for col in categorical:
diff = list(set(df.loc[testdex, col].unique())- set(df.loc[traindex,col].unique()))
if diff:
print("Column {} has {} unseen categories in test set".format(col, len(diff)))
 df.loc[df[col].isin(diff),col] = 999
if df[col].dtypes == object:
df[col] = df[col].astype(str)
 lbl = preprocessing.LabelEncoder()
df[col] = pd.Series(lbl.fit_transform(df[col].values)).astype('category' )
def load_setting_file(path: str):

 with open(path)as f:
settings = yaml.safe_load(f)
 return settings
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = deterministic
def run_inference_loop(stgs, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in tqdm(loader):
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr<set_options>
categorical_index = list(range(0, len(categorical)))
 features_names = df.columns
catboost_pool = catboost.Pool(df.loc[traindex,:],
label=y,
cat_features=categorical_index)
 test_pool = catboost.Pool(data=df.loc[testdex,:],
cat_features = categorical_index)
 del df
gc.collect()
if not torch.cuda.is_available() :
device = torch.device("cpu")
 else:
device = torch.device("cuda")
 print(device )<load_pretrained>
def catboost_blackbox(max_depth, reg_lambda):
param = {
'learning_rate': 0.2,
'bagging_temperature': 0.1,
'l2_leaf_reg': reg_lambda,
'depth': int(max_depth),
'iterations' : top_boosting_rounds,
'task_type':'GPU',
'loss_function' : "Logloss",
'objective':'Logloss',
'eval_metric' : "AUC",
'bootstrap_type' : 'Bayesian',
'random_seed': seed,
'early_stopping_rounds' : early_stopping_rounds,
'use_best_model': False,
"verbose": False
}
modelstart= time.time()
scores = catboost.cv(catboost_pool,
param,
fold_count = 2,
stratified = True,
shuffle = True,
partition_random_seed = seed,
plot = False 
)
 runtime =(time.time() - modelstart)/60
optimise = scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-mean'] - scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-std']
optimisation_info.append([scores['test-AUC-mean'].idxmax() , optimise, runtime, param, scores['test-AUC-mean'].idxmax() ])
 return optimise
model_dir = TRAINED_MODEL
test_dir = TEST_RESIZED
test_file_list = [ 
(test_dir / f"{img_id}.png", [-1] * 11)
 for img_id in smpl_sub["StudyInstanceUID"].values]
test_loader = get_dataloaders_for_inference(test_file_list, batch_size=64)
 test_preds_arr = np.zeros(( N_FOLD, len(smpl_sub), N_CLASSES))
for fold_id in FOLDS:
print(f"[fold {fold_id}]")
 stgs = load_setting_file(model_dir / f"fold{fold_id}" / "settings.yml")
 stgs["model"]["params"]["pretrained"] = False
model = MultiHeadModel(**stgs["model"]["params"])
 model_path = model_dir / f"best_model_fold{fold_id}.pth"
model.load_state_dict(torch.load(model_path, map_location=device))
test_pred = run_inference_loop(stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del model
torch.cuda.empty_cache()
gc.collect()<save_to_csv>
parameter_bounds = {
'reg_lambda':(0.1, 10),
'max_depth':(3,16)
 }
init_points = 2
n_iter = 8
optimisation_info = []
CATBOOST_BO = BayesianOptimization(catboost_blackbox,
parameter_bounds,
random_state=seed )
if CONVERT_TO_RANK:
test_preds_arr = test_preds_arr.argsort(axis=1 ).argsort(axis=1)
 sub = smpl_sub.copy()
sub[CLASSES] = test_preds_arr.mean(axis=0)
 sub.to_csv("submission.csv", index=False )<import_modules>
with timer("Bayesian Optimisation - {} Iterations".format(init_points + n_iter)) :
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 CATBOOST_BO.maximize(init_points = init_points,
n_iter = n_iter,
acq = 'ucb',
xi = 0.0,
alpha = 1e-6 )
import numpy as np
import pandas as pd
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
print("Best Score: {}".format(CATBOOST_BO.max['target']))
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<set_options>
optimisation_pd = pd.DataFrame(optimisation_info, columns = ['Best Round', 'Score', 'Runtime','Param', 'Iterations'])
 optimisation_pd.head()
warnings.filterwarnings("ignore")
<define_variables>
best_param = optimisation_pd.loc[optimisation_pd['Score'].idxmax() ,'Param']
best_param['iterations'] = top_boosting_rounds*3
best_param['learning_rate'] = 0.04
best_param['early_stopping_rounds'] = early_stopping_rounds
best_param
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
with timer("Catboost Single Model"):
best_param['iterations'] = best_iteration
model = catboost.CatBoostClassifier(**best_param)
 model.fit(catboost_pool )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128<load_from_csv>
cm = catboost.utils.get_confusion_matrix(model, catboost_pool)
 print(cm )
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )<concatenate>
roc_curve_values = catboost.utils.get_roc_curve(model, catboost_pool, plot=True )
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )<train_model>
( thresholds, fnr)= catboost.utils.get_fnr_curve(curve=roc_curve_values, plot=True )
print('Training Data:',train.info() , end = "
")
 print('Testing Data:',test.info() )<feature_engineering>
results = model.predict_proba(test_pool)[:, 1]
submission = pd.DataFrame({'id': testdex, 'target': results})
 submission.to_csv('submission.csv', index=False )
<categorify><EOS> 
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))