Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
pd.set_option('display.max_columns', 60)
 pd.set_option('display.max_rows', 100 )
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC', 'Percent']] 
.rename({'FVC': 'base_FVC', 'Percent':'base_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index()
)
!pip install -q efficientnet 
<import_modules>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
print("Tensorflow version " + tf.__version__ )<create_dataframe>
FE = list(data.Sex.unique())+ list(data.SmokingStatus.unique())
 data = pd.concat([
data,
pd.get_dummies(data.Sex),
pd.get_dummies(data.SmokingStatus)
 ], axis=1 )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('TPU Master ', tpu.master)
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print('Replicas', strategy.num_replicas_in_sync)
 DATASET1 = 'melanoma-768x768'
GCS_PATH1 = KaggleDatasets().get_gcs_path(DATASET1 )<load_from_csv>
def Normalization(df):
def get_fillness(series):
return(series - series.min())/(series.max() - series.min())
 df['Age'] = get_fillness(df['Age'])
 df['base_FVC'] = get_fillness(df['base_FVC'])
 df['base_week'] = get_fillness(df['base_week'])
 df['base_Percent'] = get_fillness(df['base_Percent'])
 return df
FE += ['Age','base_FVC','base_week','base_Percent']
data = Normalization(data )
train_details = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 train_details.isna().sum()<feature_engineering>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train_details['sex'] = train_details['sex'].fillna('male')
 train_details['age_approx'] = train_details['age_approx'].fillna(train_details['age_approx'].mean())
 train_details['anatom_site_general_challenge'] = train_details['anatom_site_general_challenge'].fillna('head/neck' )<categorify>
class Mish(Activation):
def __init__(self, activation, **kwargs):
super(Mish, self ).__init__(activation, **kwargs)
 self.__name__ = 'Mish'
def mish(inputs):
return inputs * tf.math.tanh(tf.math.softplus(inputs))
get_custom_objects().update({'Mish': Mish(mish)} )
enc1 = LabelEncoder()
enc2 = LabelEncoder()
train_details['sex'] = enc1.fit_transform(train_details['sex'])
 train_details['anatom_site_general_challenge'] = enc2.fit_transform(train_details['anatom_site_general_challenge'])
 x_vec = train_details[['sex','age_approx','anatom_site_general_challenge']]<define_variables>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="Mish", name="d1" )(z)
 x = L.Dense(100, activation="Mish", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN")
 model.compile(loss=mloss(0.8),
optimizer=tf.keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
SEED = 42
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
SIZE1 = [768,768]
LR = 0.00004
EPOCHS = 12
WARMUP = 5
WEIGHT_DECAY = 0
LABEL_SMOOTHING = 0.05
TTA = 4<define_variables>
def calc_cv_score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return -np.mean(metric )
np.random.seed(SEED)
 tf.random.set_seed(SEED )<prepare_x_and_y>
%%time
cnt = 0
BATCH_SIZE =256
EPOCHS = 1500
NFOLD = 11
kf = GroupKFold(n_splits=NFOLD)
 y = tr['FVC'].values.astype('float32')
 z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
for tr_idx, val_idx in kf.split(z, y, tr['Patient']):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 es = tf.keras.callbacks.EarlyStopping(monitor='val_loss',patience=200, min_delta = 0.000001,
verbose=1, mode='min')
 lr_sch = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.4, patience=50,
verbose=0, mode='min', min_delta=0.000001, cooldown=0, min_lr=0)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
callbacks = [es, lr_sch],
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print(calc_cv_score(y[val_idx].reshape(-1, 1), pred[val_idx]))
print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("CV SCORE", calc_cv_score(y.reshape(-1, 1), pred))
train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')
 test_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec' )<split>
tr['FVC_pred'] = pred[:, 1]
tr['Confidence_pred'] = pred[:, 2] - pred[:, 0]
df_last_3 = tr.groupby('Patient' ).tail(3 ).reset_index(drop=True)
 X = df_last_3[['Weeks', 'FVC', 'FVC_pred', 'Confidence_pred']].values
C = 0
def calc_tunned_score(y_true, y_pred, Conf):
sigma = Conf
fvc_pred = y_pred
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(y_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip*sq2)
 return -np.mean(metric)
 def objective(trial, X, y):
a = trial.suggest_uniform('a', 0, 15)
 b = trial.suggest_uniform('b', -100, 100)
 y = a * X[:, 0] + b
New_Confidence = X[:, 3] + y
return calc_tunned_score(X[:, 1], X[:, 2], New_Confidence)
 n_trials = 500
obj = partial(objective, X=X, y=C)
 study = optuna.create_study(direction="maximize")
 optuna.logging.disable_default_handler()
study.optimize(obj, n_trials=n_trials )
train_filenames1, valid_filenames1 = train_test_split(train_filenames1, test_size = 0.15, random_state = SEED )<normalization>
print('last 3 score befor tuning', calc_tunned_score(X[:, 1], X[:, 2], X[:, 3]))
print('last 3 score after tuning', study.best_value)
 param = {k:v for k,v in study.best_params.items() }
print('param', param)
 print('Training data score', calc_tunned_score(tr['FVC'].values, tr['FVC_pred'].values, tr['Confidence_pred'].values+param['a']*tr['Weeks'].values+param['b']))
def decode_image(image):
image = tf.image.decode_jpeg(image, channels = 3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*SIZE1, 3])
 return image
def data_augment(image, label = None, seed = SEED):
image = tf.image.rot90(image, k = np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed = SEED)
 image = tf.image.random_flip_up_down(image, seed = SEED)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)}
example = tf.io.parse_single_example(example, LFormat)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)}
example = tf.io.parse_single_example(example, UFormat)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset =(tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO ).with_options(ignore_order ).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<define_variables>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
unc = unc +(param['a'] * tr['Weeks'] + param['b'])
 sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train1 =(load_dataset(train_filenames1 ).map(data_augment, num_parallel_calls = AUTO ).shuffle(SEED ).batch(BATCH_SIZE, drop_remainder = True ).repeat().prefetch(AUTO))
valid_dataset1 =(load_dataset(valid_filenames1, labeled=True ).batch(BATCH_SIZE ).repeat().prefetch(AUTO))<choose_model_class>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1', 'Weeks']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1'] + param['a'] * subm.loc[~subm.FVC1.isnull() ,'Weeks'] + param['b']
<count_values><EOS> 
otest = pd.read_csv(f"{ROOT}/test.csv")
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )