Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pandas_profiling import ProfileReport
sample_size = 800000<compute_test_metric>
train=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')
 print('Train Data:')
 print(train.head())
 test=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')
 print('
Test Data:')
 print(test.head())
 sub=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print('
Submission File:')
 print(sub.head() )
def matthews_correlation(y_true, y_pred):
y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<compute_test_metric>
data_dir='.. /input/osic-pulmonary-fibrosis-progression/train'
patients=os.listdir(data_dir)
 labels_df=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)
 labels_df.head()
def my_mat_cor(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
numerator =(tp * tn - fp * fn)
 denominator =(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)) **.5
return numerator /(denominator + 1e-15)
<load_from_csv>
c=0
for patient in patients:
try:
label=labels_df.loc[patient,'FVC']
path=data_dir+'/'+patient
slices=[dicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
print(len(slices),slices[0].pixel_array.shape)
 c+=1
if c==5:
break
except:
continue
df_train = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head(9 )<define_search_space>
min_s=9999
max_s=0
for patient in patients[:]:
label=labels_df.loc[patient,'FVC']
path=data_dir+'/'+patient
slices=[len(s)for s in os.listdir(path)]
if len(slices)<min_s:
min_s=len(slices)
 if len(slices)>max_s:
max_s=len(slices)
 print('Minimum number of scans for any patient:',min_s)
 print('Maximum number of scans for any patient:',max_s )
!ls.. /input/vsb-wavelet-denoised
parq_sample_dn = pq.read_pandas('.. /input/vsb-wavelet-denoised/train_dn.parquet', columns=[str(i)for i in range(0, 6)] ).to_pandas()
parq_sample = pq.read_pandas('.. /input/vsb-power-line-fault-detection/train.parquet', columns=[str(i)for i in range(0, 6)] ).to_pandas()<drop_column>
def load_scan(path):
slices = [dicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 for s in slices:
s.SliceThickness = slice_thickness
return slices
del parq_sample_dn, parq_sample<categorify>
def get_pixels_hu(slices):
image = np.stack([s.pixel_array for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16 )
max_num = 127
min_num = -128
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
drop=train[train.duplicated(subset=['Patient','Weeks'],keep='last')]
print('No.of rows to be dropped:',drop.shape[0])
 train.drop_duplicates(subset=['Patient','Weeks'],keep='last',inplace=True )
def transform_ts(ts, n_dim=FEATURE_TIME_STEPS, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<train_model>
sub[['Patient','Weeks']]=sub.Patient_Week.str.split("_",expand = True)
 sub=sub[['Patient','Weeks','Confidence','Patient_Week']]
sub.head()
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/vsb-wavelet-denoised/train_dn.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<load_pretrained>
sub=sub.merge(test.drop('Weeks',axis = 1),on="Patient")
 sub.head()
X_orig = np.load('.. /input/5-fold-lstm-attention-fully-commented-0-694/X.npy')
 print(X_orig.shape )<train_model>
train['Dataset']='train'
sub['Dataset']='test'
data=train.append([sub])
 data.reset_index(inplace = True,drop=True)
 data.head()
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x0 = Bidirectional(CuDNNLSTM(HID, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(HID, return_sequences=True))(x0)
 x1 = GlobalAveragePooling1D()(x)
 x2 = GlobalMaxPooling1D()(x)
 x = Concatenate()([x1,x2])
 x = Dense(64, activation="relu" )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=LR), metrics=[matthews_correlation])
 return model<string_transform>
data = pd.concat([
data,
pd.get_dummies(data.Sex),
pd.get_dummies(data.SmokingStatus)
 ],axis=1)
 data.drop(['Sex','SmokingStatus'],axis=1,inplace=True)
 data['Weeks']=data['Weeks'].astype('int64')
 data.head()
splits = list(StratifiedKFold(n_splits=N_SPLITS ).split(X, y))
preds_val_list = []
y_val_list = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 csv_logger = CSVLogger('trainlog.csv', append=True, separator=',')
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx),
save_best_only=True, save_weights_only=True,
verbose=1, monitor='val_matthews_correlation',
mode='max')
 model.fit(train_X, train_y, batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=[val_X, val_y],
callbacks=[ckpt,csv_logger], verbose=0)
 model.load_weights('weights_{}.h5'.format(idx))
preds_1fold = model.predict(val_X, batch_size=512)
 result = threshold_search(val_y, preds_1fold[:,0])
 print('optimized threshold is %.2f and have score %.4f' % 
(result['threshold'], result['matthews_correlation']))
preds_val_list.append(preds_1fold)
 y_val_list.append(val_y)
 preds_val = np.concatenate(preds_val_list)[...,0]
y_val = np.concatenate(y_val_list)
 print(preds_val.shape, y_val.shape)
<load_from_csv>
def get_baseline(df):
_df=df.copy()
_df['min_week']=_df['Weeks']
_df.loc[_df.Dataset=='test','min_week']=0
_df["min_week"]=_df.groupby('Patient')['Weeks'].transform('min')
 _df['baselined_week']=_df['Weeks']-_df['min_week']
return _df
data['Weeks']=data['Weeks'].astype('int64')
 data=get_baseline(data)
 data.head()
%%time
meta_test = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_test.csv' )<drop_column>
def get_baseline_FVC(df):
_df = df.copy()
base = _df.loc[_df.Weeks == _df.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','base_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb == 1]
base.drop('nb', axis = 1, inplace = True)
 _df = _df.merge(base, on = 'Patient', how = 'left')
 _df.drop(['min_week'], axis = 1)
 return _df
data=get_baseline_FVC(data)
 data.head()
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<concatenate>
def scaling(series):
return(series-series.min())/(series.max() -series.min())
 data['Age']=scaling(data['Age'])
 data['Percent']=scaling(data['Percent'])
 data['baselined_week']=scaling(data['baselined_week'])
 data['base_FVC']=scaling(data['base_FVC'])
 data.head()
X_test_dn = np.asarray([np.concatenate([X_test_raw[i][3],X_test_raw[i+1][3], X_test_raw[i+2][3]], axis=1)for i in range(0,len(X_test_raw), 3)])
 np.save("X_test_dn.npy",X_test_dn)
 X_test_dn.shape<load_pretrained>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):

 tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype = tf.float32))
metric =(delta / sigma_clip)* sq2 + tf.math.log(sigma_clip * sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):

 qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype = tf.float32)
 e = y_true - y_pred
v = tf.maximum(q * e,(q-1)* e)
 return K.mean(v)
 def mloss(_lambda):

 def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)* score(y_true, y_pred)
 return loss
X_test_orig = np.load('.. /input/5-fold-lstm-attention-fully-commented-0-694/X_test.npy')
 print(X_test_orig.shape )<compute_train_metric>
def make_model(nh):
z = Input(( nh,), name="Patient")
 x = Dense(100, activation="elu", name="d1" )(z)
 x = Dense(100, activation="elu", name="d3" )(x)
 p1 = Dense(3, activation="linear", name="p1" )(x)
 p2 = Dense(3, activation="elu", name="p2" )(x)
 preds = Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = Model(z,preds,name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
result = threshold_search(y_val, preds_val)
 best_threshold = result['threshold']
print(result['threshold'],result['matthews_correlation'] )<predict_on_test>
features_list=['baselined_week', 'Percent', 'Age', 'base_FVC', 'Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']
train=data.loc[data.Dataset == 'train']
sub=data.loc[data.Dataset == 'test']
y=train['FVC'].values.astype(float)
 X_train=train[features_list].values
X_test=sub[features_list].values
n_rows=X_train.shape[1]
train_preds=np.zeros(( X_train.shape[0], 3))
test_preds=np.zeros(( X_test.shape[0], 3))
preds_test_prob = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test, batch_size=300, verbose=1)
 pred_3 = np.repeat(pred,3)
 preds_test_prob.append(pred_3)
<normalization>
reduce_lr_loss=tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.4,patience=150,verbose=0,epsilon=1e-4,mode='min')
 NFOLD = 6
kf = KFold(n_splits=NFOLD)
 OOF_val_score=[]
cnt = 0
BATCH_SIZE=128
EPOCHS = 800
for tr_idx, val_idx in kf.split(X_train):
cnt += 1
print(f"FOLD {cnt}")
 model=make_model(n_rows)
 history=model.fit(X_train[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_train[val_idx], y[val_idx]), verbose=0, callbacks=[reduce_lr_loss])
 print("train", model.evaluate(X_train[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", model.evaluate(X_train[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 train_preds[val_idx]=model.predict(X_train[val_idx],batch_size=BATCH_SIZE, verbose=0)
 OOF_val_score.append(model.evaluate(X_train[val_idx], y[val_idx], verbose = 0, batch_size = BATCH_SIZE, return_dict = True)['score'])
 print("predict test...")
 test_preds+=model.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/NFOLD
preds_test_prob = np.squeeze(np.mean(preds_test_prob, axis=0))
np.save('preds_test_prob.npy',preds_test_prob)
 print(preds_test_prob.shape )<data_type_conversions>
np.mean(OOF_val_score )
preds_test =(preds_test_prob > best_threshold ).astype(np.int)
 print(preds_test.shape)
 print(sum(preds_test))<load_from_csv>
sigma_opt = mean_absolute_error(y, train_preds[:,1])
 sigma_uncertain = train_preds[:,2] - train_preds[:,0]
sigma_mean = np.mean(sigma_uncertain)
 print(sigma_opt, sigma_mean )
submission = pd.read_csv('.. /input/vsb-power-line-fault-detection/sample_submission.csv')
 print(len(submission))
submission.head()<save_to_csv>
sub['FVC1'] = test_preds[:, 1]
sub['Confidence1'] = test_preds[:,2] - test_preds[:,0]
submission = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
submission.loc[~submission.FVC1.isnull() ].head(10 )
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<create_dataframe>
submission.loc[~submission.FVC1.isnull() ,'FVC'] = submission.loc[~submission.FVC1.isnull() ,'FVC1']
if sigma_mean < 70:
submission['Confidence'] = sigma_opt
else:
submission.loc[~submission.FVC1.isnull() ,'Confidence'] = submission.loc[~submission.FVC1.isnull() ,'Confidence1']
submission25 = submission.copy()
submission30 = submission.copy()
submission40 = submission.copy()
submission55 = submission.copy()
submission60 = submission.copy()<save_to_csv>
org_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(org_test)) :
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70
<define_variables><EOS> 
submission[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index = False )