Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
import numpy as np
import pandas as pd
import os
import random
from tqdm.auto import tqdm
import matplotlib.pyplot as plt
import cv2
RF = RandomForestClassifier()<define_search_space>
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, RepeatedKFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
param_grid = {
"n_estimators" : [9, 18, 27, 36, 100, 150],
"max_depth" : [2,3,5,7,9],
"min_samples_leaf" : [2, 4, 6, 8]}<choose_model_class>
seed = 123
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(seed )
RF_random = RandomizedSearchCV(RF, param_distributions=param_grid, cv=5 )<train_model>
mean = -0.13859619186395233
std = 0.1546567782480763
num_cv_folds = 4
num_repeats = 50
RF_random.fit(X_train, y_train )<find_best_params>
train_data = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/train.csv")
 print(train_data.shape)
 base_df = train_data.copy() [['Patient', 'FVC', 'Percent', 'Weeks']].sort_values(['Patient','Weeks'] ).groupby('Patient' ).head(1)
 base_df.columns = ['Patient', 'base_FVC', 'base_Percent', 'base_Weeks']
train_data = train_data.merge(base_df, on='Patient')
 print(train_data.shape)
 train_data['Weeks_passed'] = train_data.Weeks-train_data.base_Weeks
train_data = train_data[train_data.Weeks_passed>0]
train_data = train_data.drop_duplicates()
print(train_data.shape)
 train_data['est_Percent'] = train_data.apply(lambda x: mean*(x.Weeks-x.base_Weeks)+ x.base_Percent, axis=1)
 train_data['est_FVC'] = train_data['base_FVC']/train_data['base_Percent']*train_data['est_Percent']
train_data = train_data[['Patient', 'Weeks', 'FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus',
'base_FVC', 'base_Percent', 'base_Weeks', 'Weeks_passed',
'est_Percent', 'est_FVC',
]].drop_duplicates()
train_data['WHERE'] = 'train'
print(train_data.shape)
 train_data.head()
best_est_RF = RF_random.best_estimator_<compute_test_metric>
sample_sub = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv")
 sample_sub['Patient'] = sample_sub['Patient_Week'].apply(lambda x: x.split("_")[0])
 sample_sub['Weeks'] = sample_sub['Patient_Week'].apply(lambda x: x.split("_")[1])
 sample_sub.head()
print('Accuracy of classifier on training set: {:.2f}'.format(RF_random.score(X_train, y_train)* 100))
print('Accuracy of classifier on test set: {:.2f}'.format(RF_random.score(X_test, y_test)* 100))<predict_on_test>
test_data = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 test_data = test_data.groupby('Patient' ).first() [['FVC', 'Percent', 'Weeks', 'Percent', 'Age', 'Sex', 'SmokingStatus',]]
test_data.columns = ['base_FVC', 'base_Percent', 'base_Weeks', 'Percent', 'Age', 'Sex', 'SmokingStatus',]
test_data = test_data.merge(sample_sub[['Patient','Weeks']], on='Patient')
 test_data['Weeks'] = test_data.Weeks.astype(int)
 test_data['Weeks_passed'] = test_data.Weeks-test_data.base_Weeks
print(test_data.shape)
 test_data['est_Percent'] = test_data.apply(lambda x: mean*(x.Weeks-x.base_Weeks)+ x.base_Percent, axis=1)
 test_data['est_FVC'] = test_data['base_FVC']/test_data['base_Percent']*test_data['est_Percent']
print(test_data.shape)
 test_data = test_data[['Patient', 'Weeks', 'Age', 'Sex', 'SmokingStatus',
'base_FVC', 'base_Percent', 'base_Weeks', 'Weeks_passed',
'est_Percent', 'est_FVC',]].drop_duplicates()
test_data['WHERE'] = 'test'
print(test_data.shape)
 test_data.head()
y_pred = best_est_RF.predict_proba(X_train)
 y_pred = y_pred[:,1]<import_modules>
def calculate_height(row):
if row['Sex'] == 'Male':
return row['base_FVC'] /(27.63 - 0.112 * row['Age'])
 else:
return row['base_FVC'] /(21.78 - 0.101 * row['Age'])
 def scale_feature(series):
return(series - series.min())/(series.max() - series.min() )
from sklearn.metrics import roc_curve, auc<predict_on_test>
all_data = train_data.append(test_data)
 all_data['Height'] = all_data.apply(calculate_height, axis=1)
 all_data = pd.concat([
all_data,
pd.get_dummies(all_data.Sex),
pd.get_dummies(all_data.SmokingStatus)
 ], axis=1)
 all_data = all_data.drop(columns=['Sex', 'SmokingStatus'])
 all_data.head()
df_test.drop('SeriousDlqin2yrs', axis=1, inplace=True)
 y_pred=best_est_RF.predict_proba(df_test)
 y_pred= y_pred[:,1]<feature_engineering>
all_data['normalized_Weeks'] = scale_feature(all_data['Weeks'])
 all_data['normalized_base_Percent'] = scale_feature(all_data['base_Percent'])
 all_data['normalized_Age'] = scale_feature(all_data['Age'])
 all_data['normalized_base_Weeks'] = scale_feature(all_data['base_Weeks'])
 all_data['normalized_base_FVC'] = scale_feature(all_data['base_FVC'])
 all_data['normalized_Weeks_passed'] = scale_feature(all_data['Weeks_passed'])
 all_data['normalized_Height'] = scale_feature(all_data['Height'])
 all_data['normalized_est_FVC'] = scale_feature(all_data['est_FVC'])
 all_data['normalized_est_Percent'] = scale_feature(all_data['est_Percent'])
 all_data.head()
df_s["Probability"]=y_pred
df_s.head()<save_to_csv>
FEATURE_COLS = ['Female', 'Male',
'Currently smokes', 'Ex-smoker', 'Never smoked',
'normalized_Weeks', 'normalized_base_Weeks', 'normalized_Weeks_passed',
'normalized_base_Percent',
'normalized_base_FVC',
'normalized_Age',
]
df_s.to_csv("submission_RF.csv",index=False )<choose_model_class>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return -K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
XGB = XGBClassifier(n_jobs=-1)
 param_grid = {
'n_estimators' :[100,150,200,250,300],
"learning_rate" : [0.001,0.01,0.0001,0.05, 0.10 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3 ],
"colsample_bytree" : [0.5,0.7],
'max_depth': [3,4,6,8]
}<choose_model_class>
nh = len(FEATURE_COLS)
 net = make_model(nh)
 print(net.summary() )
XGB_random = RandomizedSearchCV(XGB, param_distributions=param_grid, cv=5 )<train_model>
NOISE_ON_FEATURES = ['FVC',
'normalized_Weeks', 'normalized_base_Weeks', 'normalized_Weeks_passed',
'normalized_base_Percent', 'normalized_base_FVC',
'normalized_Age', 'normalized_Height',
]
NOISE_MAX_PCT = 0.05
noise_generator = lambda x: x*(1+(np.random.randint(-NOISE_MAX_PCT*1000, NOISE_MAX_PCT*1000)/1000))
N_ARTIFICIAL_SAMPLES = len(tr )
XGB_random.fit(X_train,y_train )<find_best_params>
BATCH_SIZE = 128
EPOCHS = 200
best_est_XGB = XGB_random.best_estimator_<compute_test_metric>
pred = np.zeros(( len(tr), 3))
pe = np.zeros(( len(te[FEATURE_COLS]), 3))
rkf = RepeatedKFold(n_splits=num_cv_folds, n_repeats=num_repeats, random_state=seed)
 fold = 0
for tr_idx, val_idx in rkf.split(tr):
print(f"FOLD {fold}")
 X_tr = tr.loc[tr_idx]
X_tr_artif = X_tr.sample(n=N_ARTIFICIAL_SAMPLES, replace=True)
 X_tr_artif[NOISE_ON_FEATURES] = X_tr_artif[NOISE_ON_FEATURES].apply(noise_generator)
 X_tr = X_tr.append(X_tr_artif)
 y_tr = X_tr['FVC'].values
X_tr = X_tr[FEATURE_COLS].values
X_val = tr.loc[val_idx]
y_val = X_val['FVC'].values
X_val = X_val[FEATURE_COLS].values
X_te = te[FEATURE_COLS].values
net = make_model(nh)
 net.fit(X_tr, y_tr,
batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_val, y_val), verbose=0)
 print("train", net.evaluate(X_tr, y_tr, verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(X_val, y_val, verbose=0, batch_size=BATCH_SIZE))
print("predict val...", end=" ")
 pred[val_idx] += net.predict(X_val, batch_size=BATCH_SIZE, verbose=0)
 print("done")
 print("predict test...", end=" ")
 pe += net.predict(X_te, batch_size=BATCH_SIZE, verbose=0)/(num_cv_folds*num_repeats)
 print("done")
 fold+=1
pred = pred/num_repeats
print('Accuracy of classifier on training set: {:.2f}'.format(XGB_random.score(X_train, y_train)* 100))
print('Accuracy of classifier on test set: {:.2f}'.format(XGB_random.score(X_test, y_test)* 100))<import_modules>
X = tr[FEATURE_COLS].values
y = tr['FVC'].values
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
from imblearn.over_sampling import SMOTE<train_model>
tr['FVC_pred'] = pred[:, 1]
tr['Confidence'] = pred[:,2] - pred[:, 0]
smote = SMOTE(random_state=0)
 s_X,s_y=smote.fit_sample(X_train,y_train )<train_model>
def lb_metric(train):
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -np.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(np.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
return score
score = lb_metric(tr)
 print(f'Local Score: {score}' )
RF_random.fit(s_X,s_y )<find_best_params>
sub = te[['Patient', 'Weeks', 'base_FVC', 'base_Weeks']].copy()
sub['FVC'] = pe[:, 1]
c =(sub.loc[sub.base_Weeks == sub.Weeks, 'base_FVC']/sub.loc[sub.base_Weeks == sub.Weeks, 'FVC'] ).mean()
sub['FVC'] = c*pe[:, 1]
sub['Patient_Week'] = sub['Patient'] +"_"+sub['Weeks'].astype(str)
 sub.loc[sub.base_Weeks == sub.Weeks, 'FVC'] = sub.loc[sub.base_Weeks == sub.Weeks, 'base_FVC']
sub['Confidence'] = pe[:, 2] - pe[:, 0]
sub.loc[sub.base_Weeks == sub.Weeks, 'Confidence'] = 70
sub['Confidence'] = sub.Confidence.fillna(70 ).apply(lambda x: np.clip(x, 70, best_x+140))
sub = sub[['Patient_Week','FVC','Confidence']]
sub.head(20 )
best_est_RF1 = RF_random.best_estimator_<compute_test_metric>
sub.to_csv('submission.csv', index=False )
<predict_on_test><EOS> 
sub.to_csv('submission.csv', index=False )