Kaggle competition: OSIC Pulmonary Fibrosis Progression
ensemble1 =(sub0['target'] + sub1['target'] + sub2['target'] + sub3['target'] + sub4['target'] + sub6['target'])/6
ensemble_img1 = sub1['image_name']
ensemble_sub1 = pd.concat([ensemble_img1, ensemble1], axis = 1)
 ensemble_sub1.to_csv('submission7.csv', index=False)
 ensemble_sub1.head()<install_modules>
import numpy as np
import pandas as pd
import seaborn as sns
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, GroupKFold
!pip install -q efficientnet
!pip install -q pyyaml h5py 
<load_from_csv>
import tensorflow as tf
import tensorflow.keras.backend as backend
import tensorflow.keras.layers as layers
import tensorflow.keras.models as models
train_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<define_variables>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 return seed
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH_ORIGINAL = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 GCS_PATH_MERGED = KaggleDatasets().get_gcs_path('512x512-melanoma-tfrecords-70k-images')
 EPOCHS = 20
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
TRAINING = True
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH_MERGED + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH_MERGED + '/test*.tfrec')
 CLASSES = [0,1]
print('Training filenames
', list(TRAINING_FILENAMES))
print('Test file names
', list(TEST_FILENAMES))<define_variables>
path = ".. /input/osic-pulmonary-fibrosis-progression"
train = pd.read_csv(f"{path}/train.csv")
 test = pd.read_csv(f"{path}/test.csv")
 print('****training data head 1 value****
')
 print(train.head(1))
print('
****test data head 1 value****
')
 print(test.head(1))
VALIDATION_FILENAMES = TRAINING_FILENAMES[0:5]
TRAINING_FILENAMES = TRAINING_FILENAMES[5:]<categorify>
print('train_data_shape', train.shape)
 print('test_data_shape', test.shape)
 print('duplicates',train.duplicated().sum())
 print('duplicates',train.duplicated(subset=['Patient','Weeks'] ).sum())
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'] )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def visualize_training(history, lw = 3):
plt.figure(figsize=(10,6))
plt.plot(history.history['accuracy'], label = 'training', marker = '*', linewidth = lw)
 plt.plot(history.history['val_accuracy'], label = 'validation', marker = 'o', linewidth = lw)
 plt.title('Training Accuracy vs Validation Accuracy')
 plt.xlabel('Epochs')
 plt.ylabel('Accuracy')
 plt.legend(fontsize = 'x-large')
 plt.show()
plt.figure(figsize=(10,6))
plt.plot(history.history['loss'], label = 'training', marker = '*', linewidth = lw)
 plt.plot(history.history['val_loss'], label = 'validation', marker = 'o', linewidth = lw)
 plt.title('Training Loss vs Validation Loss')
 plt.xlabel('Epochs')
 plt.ylabel('Loss')
 plt.legend(fontsize = 'x-large')
 plt.show()
plt.figure(figsize=(10,6))
plt.plot(history.history['lr'], label = 'lr', marker = '*',linewidth = lw)
 plt.title('Learning Rate')
 plt.xlabel('Epochs')
 plt.ylabel('Learning Rate')
 plt.show()<choose_model_class>
sub = pd.read_csv(f"{path}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=8,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True )<choose_model_class>
submission = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 print('****submission_data head 1 value*****
')
 print(sub.head(1))
print('******test data head 1 value********')
 print(test.head(1))
submission.head(1 )
early_stopper = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',
patience=4,
mode='max',
baseline=None,
restore_best_weights=True)
 model_checkpoint_callback_efnB7 = tf.keras.callbacks.ModelCheckpoint(
 filepath='model_efnB7_all_four_data_best_val_acc.hdf5',
save_weights_only=True,
monitor='val_accuracy',
mode='max',
save_best_only=True)
<define_variables>
print(train.Patient.nunique() , data.Patient.nunique() , test.Patient.nunique() , submission.Patient.nunique() )
test_ds = get_test_dataset(ordered=True)
 print('Generating test file list')
 test_images_ds = test_ds.map(lambda image, idnum: image )<choose_model_class>
check_point_1 = data['min_week'] = data['Weeks']
check_point_2 = data.loc[data.WHERE=='test','min_week'] = np.nan
check_point_3 = data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 print('check_point_1
',check_point_1.head(10))
print('
check_point_2
',check_point_2)
 print('
check_point_3
',check_point_3)
 data.head(10 )
with strategy.scope() :
model_efn_b7 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
Layers.GlobalAveragePooling2D() ,
Layers.Dense(1, activation='sigmoid')
 ])
 model_efn_b7.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model_efn_b7.summary()
if TRAINING:
history_efn_b7 = model_efn_b7.fit(
 get_training_dataset() ,
epochs=EPOCHS,
callbacks=[lr_schedule, model_checkpoint_callback_efnB7, early_stopper],
steps_per_epoch=NUM_TRAINING_IMAGES // BATCH_SIZE,
validation_data=get_validation_dataset() 
)
 if TRAINING:
pd.DataFrame.from_dict(history_efn_b7.history ).to_csv('history_efn_b7.csv' , index=False)
 visualize_training(history_efn_b7)
 else:
model_efn_b7.load_weights('/kaggle/input/efficent-net-pretrained-24-epoch/model_efnB7_all_four_data_best_val_acc.hdf5')
 probabilities_efn_b7 = model_efn_b7.predict(test_images_ds)
<categorify>
base = data.loc[data.Weeks == data.min_week]
a = base
base = base[['Patient','FVC']].copy()
b = base
base.columns = ['Patient','min_FVC']
base['nb'] = 1
c = base
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 d = base
base = base[base.nb==1]
e = base
base.drop('nb', axis=1, inplace=True)
 f = base
test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
<save_to_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
data.head(10)
 del base
pred_efn_b7 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities_efn_b7)})
 pred_efn_b7.to_csv('submission_efn_b7_all_four_datasets.csv', index = False)
<save_to_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<load_from_csv>
train['age'] =(train['Age'] - train['Age'].min())/(train['Age'].max() - train['Age'].min())
 train['BASE'] =(train['min_FVC'] - train['min_FVC'].min())/(train['min_FVC'].max() - train['min_FVC'].min())
 train['week'] =(train['base_week'] - train['base_week'].min())/(train['base_week'].max() - train['base_week'].min())
 train['percent'] =(train['Percent'] - train['Percent'].min())/(train['Percent'].max() - train['Percent'].min())
 test['age'] =(test['Age'] - test['Age'].min())/(test['Age'].max() - test['Age'].min())
 test['BASE'] =(test['min_FVC'] - test['min_FVC'].min())/(test['min_FVC'].max() - test['min_FVC'].min())
 test['week'] =(test['base_week'] - test['base_week'].min())/(test['base_week'].max() - test['base_week'].min())
 test['percent'] =(test['Percent'] - test['Percent'].min())/(test['Percent'].max() - test['Percent'].min())
 submission['age'] =(submission['Age'] - submission['Age'].min())/(submission['Age'].max() - submission['Age'].min())
 submission['BASE'] =(submission['min_FVC'] - submission['min_FVC'].min())/(submission['min_FVC'].max() - submission['min_FVC'].min())
 submission['week'] =(submission['base_week'] - submission['base_week'].min())/(submission['base_week'].max() - submission['base_week'].min())
 submission['percent'] =(submission['Percent'] - submission['Percent'].min())/(submission['Percent'].max() - submission['Percent'].min())
 FE += ['age','percent','week','BASE']
f = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')[['image_name']]
n = f.shape[0]
cols = {}
files = [os.path.join('/kaggle/input/melanoma', file)for file in os.listdir('/kaggle/input/melanoma')] + [os.path.join('/kaggle/input/melanoma-public', file)for file in os.listdir('/kaggle/input/melanoma-public')]
for i, filename in enumerate(files):
cols[filename] = f'target_{i}'
ff = pd.read_csv(filename)
 ff.columns = ['image_name', f'target_{i}']
ff[f'target_{i}'] = rankdata(ff[f'target_{i}'].values.tolist())
 ff[f'target_{i}'] = ff[f'target_{i}'].apply(lambda x:(x-1)/(n-1))
f = f.merge(ff, on='image_name' )<feature_engineering>
SEED = seed_everything(42)
 NFOLD = 4
BATCH_SIZE = 128
EPOCHS = 400
f['target'] = f[cols.values() ].mean(axis=1)
 f[['image_name', 'target']].head()<save_to_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def Laplace_log_Likelihood_score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric = -(delta / sigma_clip)*sq2 - tf.math.log(sigma_clip* sq2)
 return backend.mean(metric )
f[['image_name', 'target']].to_csv('submission.csv', index=False )<compute_test_metric>
def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return backend.mean(v )
def calculate_mse(y, df):
for col in cols.values() :
df[col] =(df[col]-y)**2
return df.sum(axis=1)
 dic_errors = {}
for file, col in cols.items() :
y = f[col]
error = calculate_mse(y,f[cols.values() ])
 dic_errors[file] = error<create_dataframe>
def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*Laplace_log_Likelihood_score(y_true, y_pred)
 return loss
err_df = pd.DataFrame(dic_errors ).transpose().sum(axis=1 ).sort_values(ascending=False)
 err_df<define_variables>
def make_model() :
z = layers.Input(( 9,), name="Patient")
 x = layers.Dense(100, activation="relu", name="d1" )(z)
 x = layers.Dense(100, activation="relu", name="d2" )(x)
 x = layers.Dropout(0.002 )(x)
 p1 = layers.Dense(3, activation="linear", name="p1" )(x)
 p2 = layers.Dense(3, activation="relu", name="p2" )(x)
 preds = layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = models.Model(z, preds, name="ANN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9,
beta_2=0.999, epsilon=None, decay=0.009, amsgrad=False), metrics=[Laplace_log_Likelihood_score])
 return model
N=4<split>
y = train['FVC'].values
z = train[FE].values
ze = submission[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
biggest_error = err_df.index.tolist() [:N]<save_to_csv>
%%time
temp_val = []
temp_train = []
cnt = 0
for train_idx, val_idx in gkf.split(z, groups=train['Patient']):
cnt += 1
print(f"FOLD {cnt}")
 model.fit(z[train_idx], y[train_idx], batch_size=BATCH_SIZE, epochs= EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 training = model.evaluate(z[train_idx], y[train_idx], verbose=0, batch_size=BATCH_SIZE)
 validation = model.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE)
 print('training', training)
 print('validation', validation)
 temp_train.append(training)
 temp_val.append(validation)
 pred[val_idx] = model.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 pe += model.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += model.predict(z)/ NFOLD
f[f'target_wo_{N}'] = f[[c for k,c in cols.items() if k not in biggest_error]].mean(axis=1)
 f[['image_name', f'target_wo_{N}']].to_csv(f'sub_wo_{N}.csv', index=False, header=['image_name', 'target'])
 name_col = f'target_wo_{N}'<create_dataframe>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 logL_Score = np.mean(score)
 print(np.mean(score))
min_dist = pd.DataFrame(dic_errors ).idxmin(axis=1)
 min_vals = []
for i, sub in min_dist.iteritems() :
min_vals.append(f.loc[i,cols[sub]] )<feature_engineering>
mean_absolute_error = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
unc_mean = np.mean(unc )
f['target_arg_min'] = min_vals<save_to_csv>
stats = pd.DataFrame()
index = 0
f[['image_name', 'target_arg_min']].to_csv('sub_argmin.csv', index=False, header=['image_name', 'target'] )<save_to_csv>
data = [[index, logL_Score, mean_absolute_error, unc.mean() , unc.min() , unc.max() ,(unc>=0 ).mean() , BATCH_SIZE, EPOCHS, NFOLD, SEED]]
columns = ['Run Kernal','logL_Score', 'mean_abs_err', 'unc.mean', 'unc.min', 'unc.max', '(unc>=0 ).mean','batch_size', 'epochs', 'NFOLD','seed']
kernal_stats = pd.DataFrame(data, columns=columns)
 kernal_stats
vals = []
for i, row in pd.DataFrame(dic_errors ).iterrows() :
vals.append(f.loc[i,[cols[sub] for sub in row.sort_values(ascending=False ).index.tolist() [N+1:]]].mean())
 f['target_mean_min'] = vals
f[['image_name', 'target_mean_min']].to_csv('sub_mean_min.csv', index=False, header=['image_name', 'target'] )<save_to_csv>
stats = pd.concat([stats, kernal_stats])
 stats.to_csv('kernal.csv', index = False)
 index+=1
stats
f['global_sub']= f[[name_col, 'target_mean_min']].mean(axis=1)
 f[['image_name', 'global_sub']].to_csv('global_sub.csv', index=False, header=['image_name', 'target'] )<install_modules>
print('we are using fix seed value always to avoid RANDOMIZATION(NEED TO GET SAME RESULT)')
 print('Seed value =',SEED)
 print('Number of folds =',BATCH_SIZE)
 print('Number of epochs =',EPOCHS)
 print('
mean_absolute_error =',mean_absolute_error)
 print('unc_mean =',unc.mean())
 print('unc_min =',unc.min())
 print('unc_max =',unc.max())
 print('unc_mean =',(unc>=0 ).mean() )
!pip install -q efficientnet_pytorch > /dev/null
!pip install --no-deps timm > /dev/null<set_options>
submission['FVC1'] = pe[:, 1]
submission['Confidence1'] = pe[:, 2] - pe[:, 0]
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
subm = submission[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
DATA_PATH = '.. /input/melanoma-merged-external-data-512x512-jpeg'
TEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'<normalization>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if unc_mean<70:
subm['Confidence'] = mean_absolute_error
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
def get_valid_transforms() :
return transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ])
 class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{TEST_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.resize(image,(256,256), cv2.INTER_AREA)
 if self.transforms:
image = self.transforms(image)
 return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_from_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
df_test = pd.read_csv(f'.. /input/siim-isic-melanoma-classification/test.csv', index_col='image_name')
 test_dataset = DatasetRetriever(
 image_ids=df_test.index.values,
transforms=get_valid_transforms() , 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=8,
num_workers=2,
shuffle=False,
sampler=SequentialSampler(test_dataset),
pin_memory=False,
drop_last=False,
)<choose_model_class>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )
<features_selection><EOS>