Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<prepare_output>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
result_submission = submissions[0].copy()
result_submission['target'] = 0
for submission in submissions:
result_submission['target'] += rankdata(submission['target'])/ len(submissions )<save_to_csv>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
result_submission.to_csv('submission.csv')
 result_submission['target'].hist(bins=100);<load_from_csv>
warnings.filterwarnings("ignore")
df_folds = pd.read_csv(f'{DATA_PATH}/folds_08062020.csv', index_col='image_id' )<categorify>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, image_ids, labels, transforms=None):
super().__init__()
self.image_ids = image_ids
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.resize(image,(256, 256), cv2.INTER_AREA)
 label = self.labels[index]
if self.transforms:
image = self.transforms(image)
 return image, label, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_labels(self):
return list(self.labels )<define_variables>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128
oof_predictions = []
for checkpoint_paths in [
sorted(glob('.. /input/isic-resnext50d-32x4d-public-checkpoints/*-bestauc-*.bin')) ,
sorted(glob('.. /input/isic-resnext50d-32x4d-public-checkpoints/*-bestloss-*.bin')) ,
]:
result = {'target': [], 'pred': [], 'image_name': []}
for fold_number, checkpoint_path in tqdm(enumerate(checkpoint_paths), total=len(checkpoint_paths)) :
df_val = df_folds[(df_folds['fold'] == fold_number)&(df_folds['source'] == 'ISIC20')]
validation_dataset = DatasetRetriever(
 image_ids=df_val.index.values,
labels=df_val.target.values,
transforms=get_valid_transforms() , 
)
 validation_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=8,
num_workers=2,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
drop_last=False, 
)
 checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint);
net.eval() ;
for step,(images, targets, image_names)in enumerate(validation_loader):
with torch.no_grad() :
images = images.cuda().float()
outputs = net(images)
 y_pred = nn.functional.softmax(outputs, dim=1 ).data.cpu().numpy() [:,1]
result['target'].extend(targets.numpy())
 result['pred'].extend(y_pred)
 result['image_name'].extend(image_names)
 oof_predictions.append(pd.DataFrame(result ).set_index('image_name'))<compute_test_metric>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )
result_oof = oof_predictions[0].copy()
result_oof['pred'] = 0
for oof_prediction in oof_predictions:
result_oof['pred'] += rankdata(oof_prediction['pred'])/ len(oof_predictions)
 print('-'*30)
 print(f"[OOF RocAuc]: {sklearn.metrics.roc_auc_score(result_oof['target'], result_oof['pred']):.3f}")
 print(f"[OOF AP]: {sklearn.metrics.average_precision_score(result_oof['target'], result_oof['pred']):.3f}")
 print('-'*30 )<install_modules>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
print('Training Data:',train.info() , end = "
")
 print('Testing Data:',test.info() )
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
print(train.shape, test.shape, sub.shape, data.shape)
 print(train.Patient.nunique() , test.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<load_from_csv>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<filter>
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
train_lectures[train_lectures.user_id==5382]<filter>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]<drop_column>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
del(train_lectures )<data_type_conversions>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )<data_type_conversions>
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )<groupby>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']<data_type_conversions>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
<categorify>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<groupby><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )