Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 CLASSES = [0,1]<normalization>
import tensorflow as tf
def transform_rotation(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rotation = 15.* tf.random.normal([1],dtype='float32')
 rotation = math.pi * rotation / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3])
 def transform_shear(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
shear = 5.* tf.random.normal([1],dtype='float32')
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3])
 def transform_shift(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
height_shift = 16.* tf.random.normal([1],dtype='float32')
 width_shift = 16.* tf.random.normal([1],dtype='float32')
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3])
 def transform_zoom(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
height_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 width_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(zoom_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3] )<normalization>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
def data_augment_chaotic(image, label):
p_spatial = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')
 p_spatial2 = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')
 p_pixel = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')
 p_crop = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')
 if p_spatial >=.2:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_crop >=.7:
if p_crop >=.95:
image = tf.image.random_crop(image, size=[int(HEIGHT*.6), int(WIDTH*.6), CHANNELS])
 elif p_crop >=.85:
image = tf.image.random_crop(image, size=[int(HEIGHT*.7), int(WIDTH*.7), CHANNELS])
 elif p_crop >=.8:
image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS])
 else:
image = tf.image.random_crop(image, size=[int(HEIGHT*.9), int(WIDTH*.9), CHANNELS])
 image = tf.image.resize(image, size=[HEIGHT, WIDTH])
 if p_spatial2 >=.6:
if p_spatial2 >=.9:
image = transform_rotation(image)
 elif p_spatial2 >=.8:
image = transform_zoom(image)
 elif p_spatial2 >=.7:
image = transform_shift(image)
 else:
image = transform_shear(image)
 if p_pixel >=.4:
if p_pixel >=.85:
image = tf.image.random_saturation(image, lower=0, upper=2)
 elif p_pixel >=.65:
image = tf.image.random_contrast(image, lower=.8, upper=2)
 elif p_pixel >=.5:
image = tf.image.random_brightness(image, max_delta=.2)
 else:
image = tf.image.adjust_gamma(image, gamma=.6)
 return image, label<categorify>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
DESIRED_SIZE = 256
BATCH_SIZE=128
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def decode_image_file(file,label=None):
bits = tf.io.read_file(file)
 image = tf.image.decode_jpeg(bits,channels=3)
 image = tf.cast(image,tf.float32)/255.0
image = tf.reshape(image,[*IMAGE_SIZE,3])
 if label is None:
return image
return image,label
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
def build_lrfn(lr_start=0.00001, lr_max=0.0001,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<choose_model_class>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB3(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(512, activation='relu'),
L.Dense(128,activation='relu'),
L.Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<choose_model_class>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB3(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(512, activation='relu'),
L.Dropout(0.3),
L.Dense(256, activation='relu'),
L.Dropout(0.25),
L.Dense(128, activation='relu'),
L.Dropout(0.2),
L.Dense(1, activation='sigmoid')
 ])
 model2.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model2.summary()<find_best_params>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<train_model>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
history = model.fit(
 get_training_dataset() ,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH
)<predict_on_test>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
<prepare_output>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','week','BASE']
sub1 = sub.copy()
sub2 = sub.copy()<define_variables>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )<prepare_output>
def metric(trueFVC, predFVC, predSTD):
clipSTD = np.clip(predSTD, 70 , 9e9)
 deltaFVC = np.clip(np.abs(trueFVC-predFVC), 0 , 1000)
 return np.mean(-1*(np.sqrt(2)*deltaFVC/clipSTD)- np.log(np.sqrt(2)*clipSTD))
pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})
 pred_df.head()<load_from_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
sub4 = pd.read_csv('/kaggle/input/submissionb7/submissionB7-2.csv' )<load_from_csv>
from sklearn.linear_model import Ridge, ElasticNet
from sklearn.ensemble import RandomForestClassifier
sub5 = pd.read_csv('/kaggle/input/submissionb3/submissionB3.csv' )<merge>
NFOLD = 10
kf = KFold(n_splits=NFOLD )
del sub1['target']
sub1 = sub1.merge(pred_df, on='image_name')
 sub1.head()<save_to_csv>
cnt = 0
clf = Ridge(alpha=0.05)
 pe = np.zeros(( ze.shape[0], 2))
pred = np.zeros(( z.shape[0], 2))
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 clf.fit(z[tr_idx], y[tr_idx])
 pred[val_idx, 0] = clf.predict(z[val_idx])
 pred_std = np.mean(np.abs(y[val_idx] - pred[val_idx, 0])) * np.sqrt(2)
 pred[val_idx, 1] = pred_std
print("val", metric(y[val_idx], pred[val_idx, 0], pred[val_idx, 1]))
pe[:, 0] += clf.predict(ze)/ NFOLD
pe[:, 1] += pred_std / NFOLD
print("oof", metric(y, pred[:, 0], pred[:, 1]))
sub1.to_csv('submission-B5.csv',index=False )<load_from_csv>
print("OOF uncertainty", np.unique(pred[:, 1]))
print("TEST uncertainty", np.unique(pe[:, 1]))
sub3 = pd.read_csv('/kaggle/input/subeffnetb0/submissionB0.csv' )<save_model>
sub['FVC1'] = pe[:, 0]
sub['Confidence1'] = pe[:, 1]
model.save('EffNetB5-Melanoma.h5')
<define_variables>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
DEVICE = "TPU"
BASEPATH = ".. /input/siim-isic-melanoma-classification"<install_modules>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
!pip install -q efficientnet<import_modules>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<train_on_grid><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )