Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import os, re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
import xgboost
import lightgbm
import catboost
from sklearn import ensemble, metrics, model_selection, linear_model, preprocessing, utils
import tensorflow as tf
from tensorflow import keras
def seed_everything() :
np.random.seed(yapl.config.SEED)
 tf.random.set_seed(yapl.config.SEED)
 random.seed(a=yapl.config.SEED)
 os.environ['PYTHONHASHSEED'] = str(yapl.config.SEED)
 seed_everything()<normalization>
def laplace_likelihood(y, p):
m, s = p
diff = np.minimum(1000, np.abs(y - m))
s = np.maximum(70, s)
 lik = -np.sqrt(2)* diff / s - np.log(np.sqrt(2)* s)
 return np.mean(lik)
 def laplace_likelihood_bound(y, p):
m = p
diff = np.minimum(1000, np.abs(y - m))
s = np.maximum(70, np.sqrt(2)* diff)
 lik = -np.sqrt(2)* diff / s - np.log(np.sqrt(2)* s)
 return np.mean(lik)
 def laplace_likelihood_avg(y, p):
m = p
diff = np.minimum(1000, np.abs(y - m))
s = np.sqrt(2)* metrics.mean_absolute_error(y, m)
 lik = -np.sqrt(2)* diff / s - np.log(np.sqrt(2)* s)
 return np.mean(lik )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<normalization>
def build_folds(X, y, group=None, k=5, shuffle=False, train_mask=None, valid_mask=None):
if isinstance(X, pd.DataFrame): X = X.values
if isinstance(y, pd.DataFrame): y = y.values
if isinstance(group, pd.DataFrame): group = group.values
if isinstance(train_mask, pd.DataFrame): train_mask = train_mask.values
if isinstance(valid_mask, pd.DataFrame): valid_mask = valid_mask.values
if group is None:
folds = list(model_selection.KFold(k, shuffle=True ).split(X, y))
else:
idx = utils.shuffle(np.arange(X.shape[0]))
if shuffle:
Xs, ys, groups = X.copy() [idx], y.copy() [idx], group.copy() [idx]
folds = list(model_selection.GroupKFold(k ).split(np.array(Xs), np.array(ys), np.array(groups)))
 else:
folds = list(model_selection.GroupKFold(k ).split(X, y, group))
if train_mask is not None:
for i in range(k):
folds[i] =(np.array([j for j in folds[i][0] if train_mask[j]]), folds[i][1])
 if valid_mask is not None:
for i in range(k):
folds[i] =(folds[i][0], np.array([j for j in folds[i][1] if valid_mask[j]]))
return folds
def data_augment(image, label):
image = transform(image, DIM=yapl.config.IMG_SIZE[0])
 image = tf.image.random_flip_left_right(image)
 image = tf.image.rot90(image)
 return image, label
def process_training_data(data_file):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)
 img = tf.image.decode_jpeg(data['image'], channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.reshape(img, [*yapl.config.IMG_TRAIN_SHAPE, 3])
 label = tf.cast(data['target'], tf.int32)
 return img, label
def process_test_data(data_file):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)
 img = tf.image.decode_jpeg(data['image'], channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.reshape(img, [*yapl.config.IMG_TRAIN_SHAPE, 3])
 idnum = data['image_name']
return img, idnum<choose_model_class>
class GroupMeanTransformer:
def __init__(self, grouper, cols):
self.grouper = grouper
self.cols = cols
self.generated_features = []
self.means = dict()
pass
def fit_transform(self, x, y=None):
self.fit(x, y)
 return self.transform(x)
 def fit(self, x, y=None):
for gr in self.grouper:
group_name = '_'.join(gr)if isinstance(gr, list)else gr
for col in self.cols:
self.means[(group_name, col)] = x.groupby(gr)[col].mean()
self.means[(group_name, col)].name = '%s_mean_groupby_%s' %(col, group_name)
 self.generated_features += ['%s_mean_groupby_%s' %(col, group_name)]
self.generated_features += ['delta_%s_mean_groupby_%s' %(col, group_name)]
pass
def transform(self, x):
x = x.copy()
for k in self.means.keys() :
col = k[1]
m = self.means[k]
x[m.name] = x[m.index.names].merge(m, on=m.index.names)[m.name]
x['delta_' + m.name] = x[col] - x[m.index.names].merge(m, on=m.index.names)[m.name]
return x
def build_lrfn(lr_start=0.000005, lr_max=0.000020,
lr_min=0.000001, lr_rampup_epochs=5,
lr_sustain_epochs=0, lr_exp_decay=.8):
if yapl.config.STRATEGY is not None:
lr_max = lr_max * yapl.config.STRATEGY.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
def build_lrfn_l(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
if yapl.config.STRATEGY is not None:
lr_max = lr_max * yapl.config.STRATEGY.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
def build_lrfn_2() :
lr_start = 0.000005
lr_max = 0.000020
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
if yapl.config.STRATEGY is not None:
lr_max = lr_max * yapl.config.BATCH_SIZE
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
return lrfn
def get_cosine_schedule_with_warmup(lr = 0.00004, num_warmup_steps = 5 , num_training_steps = yapl.config.EPOCHES, num_cycles=0.5):
def lrfn(epoch):
if epoch < num_warmup_steps:
return(float(epoch)/ float(max(1, num_warmup_steps)))* lr
progress = float(epoch - num_warmup_steps)/ float(max(1, num_training_steps - num_warmup_steps))
return max(0.0, 0.5 *(1.0 + math.cos(math.pi * float(num_cycles)* 2.0 * progress)))* lr
return lrfn
lrfn = get_cosine_schedule_with_warmup()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=0)
 yapl.config.CALLBACKS.append(lr_schedule )<choose_model_class>
def _transform(x, mode: str=None, df=None):
if mode is None: return x
if mode.upper() == "DIFF": return x - df.FVC_base
if mode.upper() == "DIFF_FVC": return np.where(df.n_weeks == 0, 0.0, x - df.FVC_base)
 if mode.upper() == "PERC": return x / df.FVC_base
if mode.upper() == "PERC_FVC": return np.where(df.n_weeks == 0, 1.0, x / df.FVC_base)
 if mode.upper() == "WEEKLY": return x / np.maximum(1.0, np.abs(df.n_weeks))
if mode.upper() == "SYMLOG_WEEKLY": return x / np.log(1 + np.maximum(1.0, np.abs(df.n_weeks)))
 if re.match("^LOG", mode.upper()):
gamma = re.sub("^LOG[:]*", "", mode.upper())
 gamma = float(gamma)if len(gamma)> 0 else 0.0
return np.log(gamma + x)
 if re.match("^BOXCOX", mode.upper()):
gamma = re.sub("^BOXCOX[:]*", "", mode.upper())
 gamma = float(gamma)if len(gamma)> 0 else 1.0
return(x ** gamma - 1.0)/ gamma
return None
def _inverse_transform(x, mode=None, df=None):
if mode is None: return x
if mode.upper() == "DIFF": return x + df.FVC_base
if mode.upper() == "DIFF_FVC": return np.where(df.n_weeks == 0, df.FVC_base, x + df.FVC_base)
 if mode.upper() == "PERC": return x * df.FVC_base
if mode.upper() == "PERC_FVC": return np.where(df.n_weeks == 0, df.FVC_base, x * df.FVC_base)
 if mode.upper() == "WEEKLY": return x * np.maximum(1.0, np.abs(df.n_weeks))
if mode.upper() == "SYMLOG_WEEKLY": return x * np.log(1 + np.maximum(1.0, np.abs(df.n_weeks)))
 if re.match("^LOG", mode.upper()):
gamma = re.sub("^LOG[:]*", "", mode.upper())
 gamma = float(gamma)if len(gamma)> 0 else 0.0
return np.exp(x)- gamma
if re.match("^BOXCOX", mode.upper()):
gamma = re.sub("^BOXCOX[:]*", "", mode.upper())
 gamma = float(gamma)if len(gamma)> 0 else 1.0
return(1.0 + x * gamma)**(1 / gamma)
 return None
def transform(x, mode=None, df=None):
out = x.copy()
if isinstance(mode, list):
for m in mode:
out = _transform(out, mode=m, df=df)
 if isinstance(mode, str):
out = _transform(out, mode=mode, df=df)
 return out
def inverse_transform(x, mode=None, df=None):
out = x.copy()
if isinstance(mode, list):
for m in mode[::-1]:
out = _inverse_transform(out, mode=m, df=df)
 if isinstance(mode, str):
out = _inverse_transform(out, mode=mode, df=df)
 return out
def efficientnetbx() :
return tf.keras.Sequential([
efn.EfficientNetB0(
 input_shape=(*yapl.config.IMG_TRAIN_SHAPE, 3),
weights='imagenet',
include_top=False 
),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid')
 ] )<train_model>
def feature_eng(df):
df = df.copy()
df['n_weeks'] = df['Weeks_target'] - df['Weeks_base']
df['symlog_n_weeks'] = np.sign(df['n_weeks'])* np.log(1 + np.abs(df['n_weeks']))
df['symlog_n_weeks2'] = np.sign(df['n_weeks'])* np.log(1 + np.abs(df['n_weeks'])** 2)
 df['expdecay_n_weeks'] = np.exp(-np.abs(df['n_weeks']))
df['Sex_female'] =(df['Sex'] == 'Female' ).astype('float')
 df['Smoking_ex'] =(df['SmokingStatus'] == 'Ex-smoker' ).astype(int)
 df['Smoking_currently'] =(df['SmokingStatus'] == 'Currently smokes' ).astype(int)
 return df
def fitengine(model, traindataset, valdataset = None, istraining = True):
model.compile(
 optimizer = yapl.config.OPTIMIZER,
loss = yapl.config.LOSS,
metrics = yapl.config.ACCURACY 
)
 weight_for_0 =(1 / yapl.config.NEG_SMAPLE)*(yapl.config.TOTAL_TRAIN_IMG)/2.0
weight_for_1 =(1 / yapl.config.POS_SMAPLE)*(yapl.config.TOTAL_TRAIN_IMG)/2.0
class_weight = {0: weight_for_0, 1: weight_for_1}
history = model.fit(
 traindataset,
epochs = yapl.config.EPOCHES,
steps_per_epoch = yapl.config.TOTAL_TRAIN_IMG//yapl.config.BATCH_SIZE,
callbacks = yapl.config.CALLBACKS,
validation_data = valdataset,
validation_steps = yapl.config.TOTAL_VAL_IMG//yapl.config.BATCH_SIZE,
verbose = 0 
)
 return history<install_modules>
data_folder = ".. /input/osic-pulmonary-fibrosis-progression"
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev
!pip install wtfml
!pip install efficientnet_pytorch<set_options>
df_train = pd.read_csv(os.path.join(data_folder, "train.csv")).drop_duplicates(keep=False, subset=['Patient', 'Weeks'])
 df_train['n_obs'] = df_train.groupby('Patient' ).Weeks.cumcount()
df_train['till_last'] = df_train.groupby('Patient' ).Weeks.transform('count')- 1 - df_train['n_obs']
df_train = df_train.merge(df_train.drop(['Age', 'Sex', 'Percent', 'SmokingStatus'], 1), on='Patient', suffixes=['_base', '_target'])
 cols_before_fe = df_train.columns
df_train = feature_eng(df_train)
 FEATURES = ['FVC_base', 'Percent', 'Age'] + [c for c in df_train.columns if c not in cols_before_fe]
df_train[FEATURES].head(2 )
warnings.filterwarnings("ignore" )<import_modules>
train_mask =(df_train.n_weeks > 0)& df_train.n_obs_base.between(0, 5)& df_train.till_last_target.between(0, 3)
 valid_mask =(df_train.n_weeks > 0)& df_train.n_obs_base.between(0, 0)& df_train.till_last_target.between(0, 2 )
import torch
import torch.nn as nn
import torchvision.models as models
import torchvision
import cv2
import numpy as np
import pandas as pd
import os
from torch.utils.data import DataLoader,TensorDataset,Dataset
import matplotlib.pyplot as plt
import albumentations
from sklearn import model_selection
from sklearn.metrics import roc_auc_score
from efficientnet_pytorch import EfficientNet
from wtfml.utils import EarlyStopping<load_from_csv>
df_test = pd.read_csv(os.path.join(data_folder, "test.csv")).rename(columns={'Weeks': 'Weeks_base', 'FVC': 'FVC_base'})
 df_test = df_test.assign(k=0 ).merge(pd.DataFrame({'Weeks_target': np.arange(-12, 133 + 1), 'k': 0})).drop('k', 1)
 df_test = feature_eng(df_test)
 df_test.head(2 )
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/train.csv')
 train_df.head()<split>
print(50 * "
print("
print("
print("
print(50 * "
df = train_df.sample(frac=1 ).reset_index(drop=True)
 df['kfold'] = -1
y = train_df.target.values
kf = model_selection.StratifiedKFold(n_splits=5,shuffle=True)
 idx = kf.get_n_splits(X=df,y=y)
 print(idx)
 for fold,(x,y)in enumerate(kf.split(X=df,y=y)) :
df.loc[y,'kfold'] = fold<load_from_csv>
gr = GroupMeanTransformer(['Sex', 'SmokingStatus'], ['FVC_base'])
 gr.fit(df_train)
 df_train = gr.transform(df_train)
 df_test = gr.transform(df_test)
 FEATURES += [f for f in gr.generated_features if re.match("^delta_", f)]
df = pd.read_csv('/kaggle/input/tpu-csv/train_fold_tpu.csv')
<prepare_x_and_y>
MODE = ["PERC_FVC", "LOG"]
SIGMA_MODE = ["PERC"]
X = df_train[FEATURES].copy().values
y = df_train['FVC_target'].copy().values
group = df_train['Patient'].values
X_test = df_test[FEATURES].copy().values
class CustomDataset(Dataset):
def __init__(self,path,name,target,aug):
super(CustomDataset,self ).__init__()
self.path = path
self.name = name
self.target = target
self.aug = aug
def __len__(self):
return len(self.name)
 def __getitem__(self,index):
im_name = self.name[index]
y = self.target[index]
img_path = os.path.join(self.path,im_name + '.jpg')
 img = cv2.resize(cv2.imread(img_path),dsize=(384,384))
image = self.aug(image=img)
 l = image['image']
image = np.transpose(l,(2, 0, 1)).astype(np.float32)
 return torch.tensor(image,dtype=torch.float),torch.tensor(y)
<create_dataframe>
df_train[mean_target == mean_target.max() ]
class Data_Loader() :
def __init__(self,path,name,target,aug):
self.path = path
self.name = name
self.target = target
self.aug = aug
self.dataset = CustomDataset(self.path,self.name,self.target,self.aug)
 def get(self,batch_size,shuffle,num_workers):
sampler = torch.utils.data.distributed.DistributedSampler(self.dataset,
num_replicas = xm.xrt_world_size() ,
rank = xm.get_ordinal() ,
shuffle = shuffle)
 dataloader = torch.utils.data.DataLoader(self.dataset,
batch_size=batch_size,
shuffle=False,
sampler=sampler,
num_workers=num_workers)
 return dataloader 
<choose_model_class>
pred_oof = np.nan * np.zeros(( N_FOLDS, X.shape[0]))
pred_test = np.nan * np.zeros(( N_FOLDS, X_test.shape[0]))
for i,(idxT, idxV)in enumerate(tqdm(folds)) :
model = linear_model.LassoCV()
model.fit(Z[idxT], mean_target[idxT])
 pred_oof[i] = model.predict(Z)
 pred_oof[i, idxT] = np.nan
pred_test[i] = model.predict(Z_test)
 pred_mean = inverse_transform(np.nanmean(pred_oof, 0), MODE, df_train)
 test_mean = inverse_transform(np.nanmean(pred_test, 0), MODE, df_test )
class EffNet(nn.Module):
def __init__(self,model='b4'):
super(EffNet,self ).__init__()
model_name = 'efficientnet' + model
self.feature = EfficientNet.from_pretrained("efficientnet-b4")
 self.drop = nn.Dropout(0.3)
 self.l0 = nn.Linear(1792,1)
 def forward(self,img):
batch_size = img.shape[0]
x = self.feature.extract_features(img)
 x = nn.functional.adaptive_avg_pool2d(x,1 ).reshape(batch_size,-1)
 x = self.drop(x)
 out = self.l0(x)
 return out<compute_train_metric>
pred_oof = np.nan * np.zeros(( N_FOLDS, X.shape[0]))
pred_test = np.nan * np.zeros(( N_FOLDS, X_test.shape[0]))
for i,(idxT, idxV)in enumerate(tqdm(folds)) :
model = linear_model.LinearRegression()
model.fit(Z[idxT], sigma_target[idxT])
 pred_oof[i] = model.predict(Z)
 pred_oof[i, idxT] = np.nan
pred_test[i] = model.predict(Z_test)
 pred_sigma = inverse_transform(np.nanmean(pred_oof, 0), SIGMA_MODE, df_train)
 test_sigma = inverse_transform(np.nanmean(pred_test, 0), SIGMA_MODE, df_test )
class FocalLoss(nn.Module):
def __init__(self,alpha=1,gamma=2):
super(FocalLoss,self ).__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self,preds,truth):
criterion = nn.BCEWithLogitsLoss()
logits = criterion(preds,truth.unsqueeze(-1 ).type_as(preds))
pt = torch.exp(-logits)
 focal_loss = self.alpha*(1-pt)**self.gamma*logits
return torch.mean(focal_loss )<train_model>
lll_overall = laplace_likelihood(y[valid_mask], [pred_mean[valid_mask], pred_sigma[valid_mask]])
 lllb_overall = laplace_likelihood_bound(y[valid_mask], pred_mean[valid_mask])
 y_rmse_overall = metrics.mean_squared_error(y[valid_mask], pred_mean[valid_mask])** 0.5
y_mae_overall = metrics.mean_absolute_error(y[valid_mask], pred_mean[valid_mask])
 s_rmse_overall = metrics.mean_squared_error(opt_sigma[valid_mask], pred_sigma[valid_mask])** 0.5
s_mae_overall = metrics.mean_absolute_error(opt_sigma[valid_mask], pred_sigma[valid_mask])
 _df_base = pd.DataFrame({'y': y, 'so': opt_sigma, 'p': pred_mean, 's': pred_sigma} ).merge(valid_folds, left_index=True, right_index=True)
 lllb = _df_base.groupby('fold' ).apply(lambda x: laplace_likelihood_bound(x['y'], x['p']))
lll = _df_base.groupby('fold' ).apply(lambda x: laplace_likelihood(x['y'], [x['p'], x['s']]))
y_rmse = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_squared_error(x['y'], x['p'])** 0.5)
 y_mae = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_absolute_error(x['y'], x['p']))
s_rmse = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_squared_error(x['so'], x['s'])** 0.5)
 s_mae = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_absolute_error(x['so'], x['s']))
print(" OVERALL || FOLD-WISE")
 print("Bound: %7.4f || %7.4f(+- %7.4f)" %(lllb_overall, lllb.mean() , 2 * lllb.std()))
print("Score: %7.4f || %7.4f(+- %7.4f)" %(lll_overall, lll.mean() , 2 * lll.std()))
print("yRMSE: %7.2f || %7.2f(+- %7.2f)" %(y_rmse_overall, y_rmse.mean() , 2 * y_rmse.std()))
print("y-MAE: %7.2f || %7.2f(+- %7.2f)" %(y_mae_overall, y_mae.mean() , 2 * y_mae.std()))
print("sRMSE: %7.2f || %7.2f(+- %7.2f)" %(s_rmse_overall, s_rmse.mean() , 2 * s_rmse.std()))
print("s-MAE: %7.2f || %7.2f(+- %7.2f)" %(s_mae_overall, s_mae.mean() , 2 * s_mae.std()))
def train(model,fold):
batch_t = 128
batch_v = 128
best_score = 0
device = xm.xla_device()
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 train_transpose = albumentations.Compose([
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
albumentations.Flip(p=0.5)
 ])
 valid_transpose = albumentations.Compose(
 [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ])
 image_path = '/kaggle/input/jpeg-melanoma-384x384/train/'
train_df = df[df.kfold!=fold].reset_index(drop=True)
 valid_df = df[df.kfold==fold].reset_index(drop=True)
 train_im = train_df.image_name.values.tolist()
train_y = train_df.target.values
valid_im = valid_df.image_name.values.tolist()
valid_y = valid_df.target.values
train_dataset = Data_Loader(image_path,train_im,train_y,
train_transpose ).get(batch_size=batch_t,shuffle=True,num_workers=4)
 valid_dataset = Data_Loader(image_path,valid_im,valid_y,
valid_transpose ).get(batch_size=batch_v,shuffle=False,num_workers=4)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer,
patience=3,
threshold=0.001,
mode="max" 
)
 es = EarlyStopping(patience=5, mode="max",tpu=True)
 criterion = nn.BCEWithLogitsLoss()
epochs = 15
best_score = 0
for epoch in range(epochs):
model.train()
batch = 0
for train_data,label in train_dataset:
train_data = train_data.to(device)
 label = torch.tensor(label,dtype = torch.float32)
 label = label.to(device)
 optimizer.zero_grad()
out = model(train_data)
 loss = criterion(out,label.unsqueeze(1 ).type_as(out))
batch +=1
del train_data,label
gc.collect()
if batch%100==0 : print("EPOCH {} Loss {} batch {}".format(epoch,loss.item() ,batch))
loss.backward()
xm.optimizer_step(optimizer,barrier=True)
 gc.collect()
model.eval()
preds = []
batch = 0
for valid_data,valid_label in valid_dataset:
valid_data = valid_data.to(device)
 valid_label = torch.tensor(valid_label,dtype = torch.float32)
 valid_label = valid_label.to(device)
 batch +=1
with torch.no_grad() :
out = model(valid_data)
 loss = criterion(out,valid_label.unsqueeze(1 ).type_as(out))
preds.append(out.cpu())
 if batch%50==0 : xm.master_print('Valid Loss {} batch {}'.format(loss.item() ,batch))
del valid_data,valid_label
gc.collect()
gc.collect()
pred=np.vstack(( preds)).ravel()
auc_score = roc_auc_score(valid_y.astype(np.float32),pred)
 print("EPOCH {} AUC Score {}".format(epoch,auc_score))
schedular.step(auc_score)
 es(auc_score, model, model_path=f"model_fold_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break
gc.collect()<choose_model_class>
pd.DataFrame({'LLL': lll, 'Bound': lllb} ).boxplot(figsize=(12, 4)) ;
model = EffNet()<train_model>
submission = df_test.copy() [['Patient', 'Weeks_target']]
submission['Patient_Week'] = submission['Patient'] + "_" + submission['Weeks_target'].astype('str')
 submission['FVC'] = test_mean
submission['Confidence'] = test_sigma
submission = submission.sort_values(['Weeks_target', 'Patient'])[['Patient_Week', 'FVC', 'Confidence']]
submission.head()
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = train(model,0)
 FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )<train_model>
submission.to_csv("submission.csv", index=False )
<normalization><EOS> 
submission.to_csv("submission.csv", index=False )