Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<train_model>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
train(0)
 train(1)
 train(2)
 train(3)
 train(4 )<load_from_csv>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
def predict(fold):
test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/test.csv')
 im_path = '/kaggle/input/jpeg-melanoma-384x384/test/'
batch_t = 32
model_path = '/kaggle/input/tpu-model/model_fold_'+str(fold)+'.bin'
test_im = test_df.image_name.values.tolist()
test_y = np.ones(len(test_im))
test_dataset = Data_Loader(im_path,test_im,test_y,valid=False)
 test_dataset = DataLoader(test_dataset,batch_t,shuffle=False,num_workers=4)
 device = 'cuda'
model = EffNet()
model.load_state_dict(torch.load(model_path))
model.to(device)
 model.eval()
preds = []
batch = 0
for test_data,test_label in test_dataset:
test_data = test_data.to(device)
 batch +=1
with torch.no_grad() :
out = model(test_data)
 preds.append(out.cpu())
 if batch%50==0 : print('Batch {}'.format(batch))
pred=np.vstack(( preds)).ravel()
return pred 
<predict_on_test>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
predict_1 = predict(0)
 predict_2 = predict(1)
 predict_3 = predict(2)
 predict_4 = predict(3)
 predict_5 = predict(4 )<save_to_csv>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
prediction =(predict_1+predict_2+predict_3+predict_4+predict_5)/5
submission = pd.read_csv(".. /input/jpeg-melanoma-384x384/sample_submission.csv")
 submission.loc[:,'target'] = prediction
submission.to_csv('submission.csv',index=False )<load_from_csv>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
s1 = pd.read_csv('.. /working/submission.csv')
 s2 = pd.read_csv('.. /input/new-submit/submit_fl.csv')
 s3 = pd.read_csv('.. /input/new-submit/submit_bce.csv')
 target_res = s3.target.values
target_eff = s2.target.values
target_eff_1 = s1.target.values
result =(target_res + target_eff + target_eff_1)/3
submission = pd.read_csv(".. /input/jpeg-melanoma-384x384/sample_submission.csv")
 submission.loc[:,'target'] = result
submission.to_csv('submit_1.csv',index=False )<load_from_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
sub = pd.read_csv('.. /working/submission.csv')
 sub.head()<define_variables>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
DEVICE = "TPU"
BASEPATH = ".. /input/siim-isic-melanoma-classification"<install_modules>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
!pip install -q efficientnet<import_modules>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
random.seed(a=42)
<train_on_grid>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print("REPLICAS: ", strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE 
<define_variables>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
EPOCHS = 6
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384, 384]<load_from_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 TRAINING_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
TEST_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
CLASSES = [0,1]<categorify>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
def lrfn(epoch):
LR_START = 0.000005
LR_MAX = 0.000020 * strategy.num_replicas_in_sync
LR_MIN = 0.000001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr<init_hyperparams>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, BatchNormalization, Activation, Dropout
from tensorflow.keras.regularizers import l2 
<choose_model_class>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
with strategy.scope() :
model0 = tf.keras.Sequential([
efn.EfficientNetB0(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model0.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
model1 = tf.keras.Sequential([
efn.EfficientNetB1(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model1.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB2(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model2.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
model3 = tf.keras.Sequential([
efn.EfficientNetB3(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model3.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
model4 = tf.keras.Sequential([
efn.EfficientNetB4(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model4.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
model6 = tf.keras.Sequential([
efn.EfficientNetB6(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model6.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy']
)<train_model>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 history0 = model0.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 history1 = model1.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 history2 = model2.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 history3 = model3.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 history4 = model4.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 history6 = model6.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule] )<predict_on_test>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities0 = model0.predict(test_images_ds)
 probabilities1 = model1.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 probabilities3 = model3.predict(test_images_ds)
 probabilities4 = model4.predict(test_images_ds)
 probabilities6 = model6.predict(test_images_ds )<save_to_csv>
sub['FVC1'] = 0.996*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
print('Generating submission.csv files...')
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )<create_dataframe>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
pred_df0 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities0)})
 pred_df1 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities1)})
 pred_df2 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities2)})
 pred_df3 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities3)})
 pred_df4 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities4)})
 pred_df6 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities6)})
 pred_df6.head()<prepare_output>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
sub0 = sub.copy()
sub1 = sub.copy()
sub2 = sub.copy()
sub3 = sub.copy()
sub4 = sub.copy()
sub6 = sub.copy()
sub4.head()<save_to_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
del sub0['target']
sub0 = sub0.merge(pred_df0, on='image_name')
 sub0.to_csv('submission0.csv', index=False)
 del sub1['target']
sub1 = sub1.merge(pred_df1, on='image_name')
 sub1.to_csv('submission1.csv', index=False)
 del sub2['target']
sub2 = sub2.merge(pred_df2, on='image_name')
 sub2.to_csv('submission2.csv', index=False)
 del sub3['target']
sub3 = sub3.merge(pred_df3, on='image_name')
 sub3.to_csv('submission3.csv', index=False)
 del sub4['target']
sub4 = sub4.merge(pred_df4, on='image_name')
 sub4.to_csv('submission4.csv', index=False)
 del sub6['target']
sub6 = sub6.merge(pred_df6, on='image_name')
 sub6.to_csv('submission6.csv', index=False)
 sub4.head()<save_to_csv>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )