Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
EPOCHS = 20
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384, 384]
FOLDS = 5
AUG_BATCH = BATCH_SIZE
FIRST_FOLD_ONLY = False
SEED = 777<load_from_csv>
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print('Train shape: ', df_train.shape)
 print('Number of unique customers in train: {}'.format(df_train['Patient'].nunique()))
print('Test shape:', df_test.shape )
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 TRAINING_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
TEST_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
CLASSES = [0,1]<categorify>
df_base = df_train.drop_duplicates(subset='Patient', keep='first')
 df_base = df_base[['Patient', 'Weeks', 'FVC',
'Percent', 'Age']].rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_base.head(3 )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset(dataset, do_aug=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.batch(AUG_BATCH)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset, do_onehot=False):
dataset = dataset.batch(BATCH_SIZE)
 if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<categorify>
df_train['visit'] = 1
df_train['visit'] = df_train[['Patient', 'visit']].groupby('Patient' ).cumsum()
df_train = df_train.loc[df_train['visit'] > 1, :]
def onehot(image,label):
CLASSES = 104
return image,tf.one_hot(label,CLASSES )<categorify>
df_train = pd.merge(df_train,
df_base,
on='Patient',
how='left')
 print(df_train.shape)
 df_train.head(3 )
def transform(image,label):
DIM = IMAGE_SIZE[0]
CLASSES = 104
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4<choose_model_class>
df_train['weeks_passed'] = df_train['Weeks'] - df_train['base_week']
df_train = pd.get_dummies(df_train, columns=['Sex', 'SmokingStatus'] )
def lrfn(epoch):
LR_START = 0.000005
LR_MAX = 0.000075 * strategy.num_replicas_in_sync
LR_MIN = 0.000001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<set_options>
sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 sub.head()
reg_l2 = 0.0001
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )<choose_model_class>
df_test = df_test.rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_test = pd.merge(sub,
df_test,
on='Patient',
how='right')
 df_test = pd.get_dummies(df_test, columns=['Sex', 'SmokingStatus'])
 df_test['weeks_passed'] = df_test['Weeks'] - df_test['base_week']
df_test.head()
def get_model() :
with strategy.scope() :
enet = efn.EfficientNetB3(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['binary_accuracy'] 
)
 return model
def train_cross_validate(folds = 5):
histories = []
models = []
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('
print('
print('
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model)
 histories.append(history)
 if FIRST_FOLD_ONLY: break
return histories, models
def train_and_predict(folds = 5):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training %i folds'%folds)
 histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds)for i in range(1)], axis = 0)
 else: probabilities = np.average([models[i].predict(test_images_ds)for i in range(folds)], axis = 0)
 predictions = probabilities
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
pred_df = pd.DataFrame({'image_name': test_ids, 'target': list(predictions)})
 pred_df.target = pred_df.target.map(lambda x: x[0])
 del sub['target']
sub = sub.merge(pred_df, on='image_name')
 sub.head()
sub.to_csv('submission.csv', index=False)
 return histories, models
histories, models = train_and_predict(folds = FOLDS )<choose_model_class>
missing_columns = np.setdiff1d(df_train.drop(['Patient', 'FVC', 'Percent', 'Age', 'visit'], axis = 1 ).columns, df_test.columns)
 if len(missing_columns)> 0:
print('/!\ Missing columns in test: ', missing_columns)
 for col in missing_columns:
df_test[col] = 0
"<predict_on_test>
def OSIC_metric(y_true, y_pred, y_pred_std):
delta = np.clip(abs(y_true - y_pred), 0, 1000)
 std_clipped = np.clip(y_pred_std, 70, np.inf)
 return np.mean(-(np.sqrt(2)*delta/std_clipped)- np.log(np.sqrt(2)*std_clipped))
"<save_to_csv>
class Model() :
def __init__(self, model=BayesianRidge() , n_splits=2):
self.regressor = model
self.n_splits = n_splits
self.gkf = GroupKFold(n_splits=n_splits)
 self.train_cols = ['Weeks', 'base_week', 'base_FVC',
'base_percent', 'base_age', 'weeks_passed', 'Sex_Female',
'Sex_Male', 'SmokingStatus_Currently smokes',
'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']
def fit(self, X, y):
self.regressor.fit(X, y)
 def predict(self, X):
pred = self.regressor.predict(X, return_std=True)
 return pred
def fit_predict_cv(self, df, df_test=pd.DataFrame()):
scores = np.zeros(( self.n_splits,))
oof = np.zeros(( len(df),))
oof_std = np.zeros_like(oof)
 if len(df_test)> 0:
pred_sub = np.zeros(( len(df_test), self.n_splits))
pred_sub_std = np.zeros_like(pred_sub)
 target = 'FVC'
for i,(train_idx, val_idx)in enumerate(self.gkf.split(df, groups=df['Patient'])) :
X_train = df.loc[train_idx, self.train_cols]
y_train = df.loc[train_idx, target]
X_val = df.loc[val_idx, self.train_cols]
y_val = df.loc[val_idx, target]
self.fit(X_train, y_train)
 pred_train, pred_train_std = self.predict(X_train)
 pred_val, pred_val_std = self.predict(X_val)
 if len(df_test)> 0:
pred_sub[:, i], pred_sub_std[:, i] = self.predict(df_test[self.train_cols])
 oof[val_idx] = pred_val
oof_std[val_idx] = pred_val_std
print('Train score: {0:.2f} | Test score: {1:.2f}'.format(OSIC_metric(y_train, pred_train, pred_train_std),
OSIC_metric(y_val, pred_val, pred_val_std)))
 print('OOF score: {0:.4f}'.format(OSIC_metric(df[target], oof, oof_std)))
 res = dict()
res['oof'] = oof
res['oof_std'] = oof_std
if len(df_test)> 0:
res['pred_sub'] = pred_sub.mean(axis=1)
 res['pred_sub_std'] = pred_sub_std.mean(axis=1)
 return res
"<install_modules>
fvc_model = Model()
!pip install yapl==0.1.2 efficientnet > /dev/null
!rm -rf *<import_modules>
res = fvc_model.fit_predict_cv(df_train, df_test )
import yapl
import os
import re
import random
import pandas as pd
import numpy as np
import math
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import tensorflow as tf
import tensorflow.keras.backend as K
from kaggle_datasets import KaggleDatasets
import efficientnet.tfkeras as efn
from yapl.config.config import Config
import matplotlib.pyplot as plt<set_options>
df_test['FVC'] = res['pred_sub']
df_test['Confidence'] = res['pred_sub_std']
submission = sub[['Patient_Week']]
submission = pd.merge(submission,
df_test[['Patient_Week', 'FVC', 'Confidence']],
on='Patient_Week',
how='left')
 submission.head()
tf.keras.backend.clear_session()<set_options>
submission.to_csv('submission.csv', index=False )
class configSetup(Config):
def __init__(self):
super().__init__()
self.IS_EXT = True
self.DO_VAL_SPLIT = True
self.EXPERIMENT_NAME = 'melanoma-detection-yapl-testing'
if self.IS_EXT:
self.GCS_DS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 self.TOTAL_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/train*.tfrec')
 self.TEST_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/test*.tfrec')
 self.TRAIN_CSV = '.. /input/melanoma-128x128/train.csv'
else:
self.GCS_DS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 self.TOTAL_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/train/*.tfrec')
 self.TEST_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/test/*.tfrec')
 self.TRAIN_CSV = '.. /input/siim-isic-melanoma-classification/train.csv'
self.TEST_CSV = '.. /input/siim-isic-melanoma-classification/test.csv'
self.VALIDATION_CSV = ""
self.TOTAL_TRAIN_IMG = 0
self.TOTAL_TEST_IMG = 0
self.NEG_SMAPLE = 32542
self.POS_SMAPLE = 584
self.IMG_SIZE = [384, 384]
self.IMG_TRAIN_SHAPE = [384,384]
self.IMG_SHAPE =(384, 384, 3)
 self.DO_FINETUNE = True
self.BATCH_SIZE = 16
self.EPOCHES = 10
self.SEED = 127
self.FOLDS = 3
self.LOSS = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 self.OPTIMIZER = tf.keras.optimizers.Adam()
self.ACCURACY = ['AUC']
self.CALLBACKS = [
tf.keras.callbacks.EarlyStopping(
 monitor='val_auc', restore_best_weights=True, mode = 'max', patience = 3, verbose = 1, min_delta = 0.001, 
)
 ]
self.STRATEGY = None<set_options>
submission.to_csv('submission.csv', index=False )
configSetup().make_global()
yapl.backend = 'tf'
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 yapl.config.STRATEGY = strategy
yapl.config.BATCH_SIZE = 32 * strategy.num_replicas_in_sync
else:
strategy = tf.distribute.get_strategy()
yapl.config.TOTAL_TEST_IMG = len(pd.read_csv(yapl.config.TEST_CSV ).image_name)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<load_from_csv>
<categorify><EOS> 
jovian.commit(project='osic-bayesian' )