Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<predict_on_test>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
preds_v1_test = model.predict(xgtrainTest )<save_to_csv>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
submission = pd.DataFrame(
 {'key': testKey.key, 'fare_amount': preds_v1_test },
columns = ['key', 'fare_amount'])
 submission.to_csv('submissionXGlast.csv', index = False )<import_modules>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
import numpy as np
import pandas as pd
import datetime as dt
from sklearn.model_selection import train_test_split
import xgboost as xgb
import os<load_from_csv>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
train_df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/train.csv', nrows = 6_000_000)
<drop_column>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
def clean_df(df):
return df[(df.fare_amount > 0)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -70)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.dropoff_longitude > -80)&(df.dropoff_longitude < -70)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.passenger_count > 0)&(df.passenger_count < 10)]
train_df = clean_df(train_df)
 print(len(train_df))<concatenate>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):
R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def add_airport_dist(dataset):
jfk_coord =(40.639722, -73.778889)
 ewr_coord =(40.6925, -74.168611)
 lga_coord =(40.77725, -73.872611)
 pickup_lat = dataset['pickup_latitude']
dropoff_lat = dataset['dropoff_latitude']
pickup_lon = dataset['pickup_longitude']
dropoff_lon = dataset['dropoff_longitude']
pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1])
 dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon)
 pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])
 dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon)
 pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1])
 dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)
 dataset['jfk_dist'] = pd.concat([pickup_jfk, dropoff_jfk], axis=1 ).min(axis=1)
 dataset['ewr_dist'] = pd.concat([pickup_ewr, dropoff_ewr], axis=1 ).min(axis=1)
 dataset['lga_dist'] = pd.concat([pickup_lga, dropoff_lga], axis=1 ).min(axis=1)
 return dataset
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC")
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
dataset['year'] = dataset.pickup_datetime.dt.year
return dataset
train_df = add_datetime_info(train_df)
 train_df = add_airport_dist(train_df)
 train_df['distance'] = sphere_dist(train_df['pickup_latitude'], train_df['pickup_longitude'],
train_df['dropoff_latitude'] , train_df['dropoff_longitude'])
 train_df.head() 
<drop_column>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
train_df.drop(columns=['key', 'pickup_datetime'], inplace=True)
 train_df.head()<split>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
y = train_df['fare_amount']
train = train_df.drop(columns=['fare_amount'])
 x_train,x_test,y_train,y_test = train_test_split(train,y,random_state=0,test_size=0.01 )<train_on_grid>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
params = {
'max_depth': 8,
'eta':.03,
'subsample': 1,
'colsample_bytree': 0.8,
'objective':'reg:linear',
'eval_metric':'rmse',
'silent': 1
}
CV=False
if CV:
dtrain = xgb.DMatrix(train,label=y)
 gridsearch_params = [ 
(eta)
 for eta in np.arange (.04, 0.12,.02)
 ]
min_rmse = float("Inf")
 best_params = None
for(eta)in gridsearch_params:
print("CV with eta={} ".format(
 eta))
params['eta'] = eta
cv_results = xgb.cv(
 params,
dtrain,
num_boost_round=1000,
nfold=3,
metrics={'rmse'},
early_stopping_rounds=10 
)
 mean_rmse = cv_results['test-rmse-mean'].min()
boost_rounds = cv_results['test-rmse-mean'].argmin()
if mean_rmse < min_rmse:
min_rmse = mean_rmse
best_params =(eta)
 print("Best params: {}, RMSE: {}".format(best_params, min_rmse))
else:
params['silent'] = 0
print(params)
 def XGBmodel(x_train,x_test,y_train,y_test,params):
matrix_train = xgb.DMatrix(x_train,label=y_train)
 matrix_test = xgb.DMatrix(x_test,label=y_test)
 model=xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(x_train,x_test,y_train,y_test,params)
<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
test_df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/test.csv')
 test_df = add_datetime_info(test_df)
 test_df = add_airport_dist(test_df)
 test_df['distance'] = sphere_dist(test_df['pickup_latitude'], test_df['pickup_longitude'],
test_df['dropoff_latitude'] , test_df['dropoff_longitude'])
 test_key = test_df['key']
x_pred = test_df.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit = model.best_ntree_limit )<save_to_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()<set_options>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
%matplotlib inline
plt.style.use('seaborn-whitegrid')
<load_from_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
df_train = pd.read_csv(".. /input/new-york-city-taxi-fare-prediction/train.csv", nrows=1_000_000,parse_dates=["pickup_datetime"])
 df_train.head()<load_from_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 x = L.Dense(50, activation="relu", name="d3" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 p3 = L.Dense(3, activation="relu", name="p3" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model
df_test = pd.read_csv(".. /input/new-york-city-taxi-fare-prediction/test.csv",parse_dates=["pickup_datetime"])
 df_test.head()<train_model>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
print('Old size: %d' % len(df_train))
df_train = df_train.dropna(how = 'any', axis = 'rows')
 print('New size: %d' % len(df_train))<feature_engineering>
NFOLD = 10
kf = KFold(n_splits=NFOLD )
df_train['year'] = df_train.pickup_datetime.apply(lambda t: t.year)
 df_train['month'] = df_train.pickup_datetime.apply(lambda t: t.month)
 df_train['weekday'] = df_train.pickup_datetime.apply(lambda t: t.weekday())
 df_train['hour'] = df_train.pickup_datetime.apply(lambda t: t.hour )<sort_values>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=700,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
df_train[['fare_amount', 'hour']].groupby(['hour'], as_index=False ).mean().sort_values(by='fare_amount', ascending=False )<sort_values>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
df_train[['fare_amount', 'weekday']].groupby(['weekday'], as_index=False ).mean().sort_values(by='fare_amount', ascending=False )<sort_values>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
df_train[['fare_amount', 'year']].groupby(['year'], as_index=False ).mean().sort_values(by='fare_amount', ascending=False )<compute_test_metric>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
def distance(lat1, lon1, lat2, lon2):
p = 0.017453292519943295
a = 0.5 - np.cos(( lat2 - lat1)* p)/2 + np.cos(lat1 * p)* np.cos(lat2 * p)*(1 - np.cos(( lon2 - lon1)* p)) / 2
return 12742 * np.arcsin(np.sqrt(a))<feature_engineering>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
df_train['distance'] = distance(df_train.pickup_latitude, df_train.pickup_longitude, \
df_train.dropoff_latitude, df_train.dropoff_longitude )<train_model>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<concatenate><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )