Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
model = Sequential()
model.add(Dense(256, activation='relu', input_dim=train_df_scaled.shape[1], activity_regularizer=regularizers.l1(0.01)))
 model.add(BatchNormalization())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(32, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(8, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(1))
adam = optimizers.adam(lr=LEARNING_RATE)
 model.compile(loss='mse', optimizer=adam, metrics=['mae'] )<train_model>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
print('Dataset size: %s' % DATASET_SIZE)
 print('Epochs: %s' % EPOCHS)
 print('Learning rate: %s' % LEARNING_RATE)
 print('Batch size: %s' % BATCH_SIZE)
 print('Input dimension: %s' % train_df_scaled.shape[1])
 print('Features used: %s' % train_df.columns )<train_model>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
history = model.fit(x=train_df_scaled, y=train_labels, batch_size=BATCH_SIZE, epochs=EPOCHS,
verbose=1, validation_data=(validation_df_scaled, validation_labels),
shuffle=True )<predict_on_test>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=64
prediction = model.predict(test_scaled, batch_size=128, verbose=1 )<compute_test_metric>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
output_submission(test, prediction, 'key', 'fare_amount', SUBMISSION_NAME )<load_from_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
train_df_v1 = pd.read_csv('.. /input/train.csv', nrows = 10000000 , parse_dates= ['key'] )<feature_engineering>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
def add_travel_vector_features(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
add_travel_vector_features(train_df_v1 )<feature_engineering>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
def add_date_data(df):
train_df_v1['weekday']=train_df_v1.key.apply(lambda x:calendar.day_name[x.weekday() ])
 train_df_v1['hour'] = train_df_v1.key.map(lambda x: x.hour)
 train_df_v1['day'] = train_df_v1.key.map(lambda x: x.day)
 train_df_v1['month'] = train_df_v1.key.map(lambda x: x.month)
 train_df_v1['year'] = train_df_v1.key.map(lambda x: x.year)
 add_date_data(train_df_v1 )<drop_column>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
print('Old size: %d' % len(train_df_v1))
train_df_v1 = train_df_v1.dropna(how = 'any', axis = 'rows')
 features_v1_list = ['pickup_longitude','pickup_latitude', 'dropoff_longitude',
'dropoff_latitude','passenger_count', 'abs_diff_longitude', 'abs_diff_latitude'
,'hour','day','month','year','weekday']
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_latitude'] < 1 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_longitude'] < 1 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_latitude'] > 0.001 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_longitude'] > 0.001 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_longitude'] < -72 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_longitude'] > -75 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_latitude'] < 41 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_latitude'] > 39 ]
train_df_v1 = train_df_v1.loc[train_df_v1['fare_amount'] > 1 ]
print('New size: %d' % len(train_df_v1))
train_df_v1_features = train_df_v1[features_v1_list]<categorify>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
class MultiColumnLabelEncoder:
def __init__(self,columns = None):
self.columns = columns
def fit(self,X,y=None):
return self
def transform(self,X):

 output = X.copy()
if self.columns is not None:
for col in self.columns:
output[col] = LabelEncoder().fit_transform(output[col])
 else:
for colname,col in output.iteritems() :
output[colname] = LabelEncoder().fit_transform(col)
 return output
def fit_transform(self,X,y=None):
return self.fit(X,y ).transform(X )<categorify>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
train_df_v1_features_new = MultiColumnLabelEncoder(columns = ['weekday'] ).fit_transform(train_df_v1_features)
 def dist(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
distance = np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 return distance<define_variables>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
label_v1 = train_df_v1['fare_amount']<train_model>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
def XGB_regressor(train_X, train_y, feature_names=None, seed_val=2017, num_rounds=500):
param = {}
param['objective'] = 'reg:linear'
param['eta'] = 0.05
param['max_depth'] = 8
param['silent'] = 1
param['eval_metric'] = 'rmse'
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 model = xgb.train(plst, xgtrain, num_rounds)
 return model<prepare_x_and_y>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="relu", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model
model = XGB_regressor(train_X = train_df_v1_features_new , train_y = label_v1 )<prepare_x_and_y>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
xgtrain = xgb.DMatrix(train_df_v1_features_new, label=label_v1 )<predict_on_test>
NFOLD = 10
kf = KFold(n_splits=NFOLD )
preds_v1 = model.predict(xgtrain )<compute_test_metric>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=400,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
rmse = np.sqrt(sckit.metrics.mean_squared_error(label_v1, preds_v1))
print("RMSE: %f" %(rmse))<load_from_csv>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
test_df_v1 = pd.read_csv('.. /input/test.csv' )<load_from_csv>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
testKey = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
test_df_v1_fordates = test_df_v1
test_df_v1_fordates['key']= pd.to_datetime(test_df_v1_fordates['key'] )<feature_engineering>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
test_df_v1_fordates['hour'] = test_df_v1_fordates.key.map(lambda x: x.hour)
 test_df_v1_fordates['day'] = test_df_v1_fordates.key.map(lambda x: x.day)
 test_df_v1_fordates['month'] = test_df_v1_fordates.key.map(lambda x: x.month)
 test_df_v1_fordates['year'] = test_df_v1_fordates.key.map(lambda x: x.year)
 test_df_v1_fordates['weekday']=test_df_v1_fordates.key.apply(lambda x:calendar.day_name[x.weekday() ])
 add_travel_vector_features(test_df_v1_fordates)
 test_df_v1_features = test_df_v1_fordates[features_v1_list]
test_df_v1_features = MultiColumnLabelEncoder(columns = ['weekday'] ).fit_transform(test_df_v1_features )<feature_engineering>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<prepare_x_and_y><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )