Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<feature_engineering>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
data = [df_train,df_test]
for i in data:
i["pickup_datetime"] = pd.to_datetime(i["pickup_datetime"])
 i['year'] = i['pickup_datetime'].dt.year
i['mth'] = i['pickup_datetime'].dt.month
i['date'] = i['pickup_datetime'].dt.day
i['day_of_week'] = i['pickup_datetime'].dt.dayofweek
i['hour'] = i['pickup_datetime'].dt.hour<train_on_grid>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
X = df_train.drop(columns=['fare_amount', 'pickup_datetime', 'fare_per_mile', 'key'])
 y = df_train['fare_amount'] - 2.5
gbm_param_grid = {
'colsample_bytree': [0.7],
'n_estimators': [50],
'max_depth': [5]
}
gbm = xgb.XGBRegressor()
grid_mse = GridSearchCV(estimator=gbm, param_grid=gbm_param_grid,
scoring='neg_mean_squared_error', cv=2, verbose=1)
 grid_mse.fit(X, y)
 print("Best parameters found: ", grid_mse.best_params_)
 print("Lowest RMSE found: ", np.sqrt(np.abs(grid_mse.best_score_)) )<predict_on_test>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
test_key = df_test['key']
x_pred = df_test.drop(columns=['key', 'pickup_datetime'])
 prediction = grid_mse.predict(x_pred)+ 2.5<save_to_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('./taxi_fare_submission.csv',index=False)
 submission.head()<import_modules>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
import numpy as np
import pandas as pd
from datetime import datetime
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
from keras.callbacks import EarlyStopping
from keras import optimizers
from keras import regularizers<filter>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
def clean(df):
df = df[(-76 <= df['pickup_longitude'])&(df['pickup_longitude'] <= -72)]
df = df[(-76 <= df['dropoff_longitude'])&(df['dropoff_longitude'] <= -72)]
df = df[(38 <= df['pickup_latitude'])&(df['pickup_latitude'] <= 42)]
df = df[(38 <= df['dropoff_latitude'])&(df['dropoff_latitude'] <= 42)]
df = df[(0 < df['fare_amount'])&(df['fare_amount'] <= 250)]
df = df[(df['dropoff_longitude'] != df['pickup_longitude'])]
df = df[(df['dropoff_latitude'] != df['pickup_latitude'])]
return df<data_type_conversions>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
tr.shape, chunk.shape, sub.shape
def late_night(row):
if(row['hour'] <= 6)or(row['hour'] >= 20):
return 1
else:
return 0
def night(row):
if(( row['hour'] <= 20)and(row['hour'] >= 16)) and(row['weekday'] < 5):
return 1
else:
return 0
def manhattan(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
return np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 def add_time_features(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S %Z')
 df['year'] = df['pickup_datetime'].apply(lambda x: x.year)
 df['month'] = df['pickup_datetime'].apply(lambda x: x.month)
 df['day'] = df['pickup_datetime'].apply(lambda x: x.day)
 df['hour'] = df['pickup_datetime'].apply(lambda x: x.hour)
 df['weekday'] = df['pickup_datetime'].apply(lambda x: x.weekday())
 df['pickup_datetime'] = df['pickup_datetime'].apply(lambda x: str(x))
df['night'] = df.apply(lambda x: night(x), axis=1)
 df['late_night'] = df.apply(lambda x: late_night(x), axis=1)
 df = df.drop('pickup_datetime', axis=1)
 return df
def add_coordinate_features(df):
lat1 = df['pickup_latitude']
lat2 = df['dropoff_latitude']
lon1 = df['pickup_longitude']
lon2 = df['dropoff_longitude']
df['latdiff'] =(lat1 - lat2)
 df['londiff'] =(lon1 - lon2)
 return df
def add_distances_features(df):
ny =(-74.0063889, 40.7141667)
 jfk =(-73.7822222222, 40.6441666667)
 ewr =(-74.175, 40.69)
 lgr =(-73.87, 40.77)
 lat1 = df['pickup_latitude']
lat2 = df['dropoff_latitude']
lon1 = df['pickup_longitude']
lon2 = df['dropoff_longitude']
df['euclidean'] =(df['latdiff'] ** 2 + df['londiff'] ** 2)** 0.5
df['manhattan'] = manhattan(lat1, lon1, lat2, lon2)
 df['downtown_pickup_distance'] = manhattan(ny[1], ny[0], lat1, lon1)
 df['downtown_dropoff_distance'] = manhattan(ny[1], ny[0], lat2, lon2)
 df['jfk_pickup_distance'] = manhattan(jfk[1], jfk[0], lat1, lon1)
 df['jfk_dropoff_distance'] = manhattan(jfk[1], jfk[0], lat2, lon2)
 df['ewr_pickup_distance'] = manhattan(ewr[1], ewr[0], lat1, lon1)
 df['ewr_dropoff_distance'] = manhattan(ewr[1], ewr[0], lat2, lon2)
 df['lgr_pickup_distance'] = manhattan(lgr[1], lgr[0], lat1, lon1)
 df['lgr_dropoff_distance'] = manhattan(lgr[1], lgr[0], lat2, lon2)
 return df<save_to_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="relu", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005, amsgrad=False), metrics=[score])
 return model
net = make_model()
print(net.summary())
 print(net.count_params())
def output_submission(raw_test, prediction, id_column, prediction_column, file_name):
df = pd.DataFrame(prediction, columns=[prediction_column])
 df[id_column] = raw_test[id_column]
df[[id_column, prediction_column]].to_csv(( file_name), index=False)
 print('Output complete')
 def plot_loss_accuracy(history):
plt.figure(figsize=(20,10))
plt.plot(history.history['loss'])
 plt.plot(history.history['val_loss'])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper right')
 plt.show()<define_variables>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
TRAIN_PATH = '.. /input/train.csv'
TEST_PATH = '.. /input/test.csv'
SUBMISSION_NAME = 'submission.csv'
BATCH_SIZE = 256
EPOCHS = 50
LEARNING_RATE = 0.001
DATASET_SIZE = 6000000<load_from_csv>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
datatypes = {'key': 'str',
'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
train = pd.read_csv(TRAIN_PATH, nrows=DATASET_SIZE, dtype=datatypes, usecols=[1,2,3,4,5,6])
 test = pd.read_csv(TEST_PATH )<feature_engineering>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
train = clean(train)
 train = add_time_features(train)
 test = add_time_features(test)
 add_coordinate_features(train)
 add_coordinate_features(test)
 train = add_distances_features(train)
 test = add_distances_features(test)
 train.head(5 )<drop_column>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
dropped_columns = ['pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude']
train_clean = train.drop(dropped_columns, axis=1)
 test_clean = test.drop(dropped_columns + ['key', 'passenger_count'], axis=1)
 train_clean.head(5 )<split>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10)
 subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
<normalization><EOS> 
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False)