Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<groupby>
import numpy as np
import pandas as pd
import os
import random
import matplotlib.pyplot as plt
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tqdm import tqdm
from PIL import Image
import pydicom
train = sales_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum().reset_index()<rename_columns>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
train.rename(columns={"item_cnt_day": "item_cnt_month"}, inplace=True )<feature_engineering>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE = 128
train['item_cnt_month'] = train['item_cnt_month'].fillna(0 ).clip(0,20 )<count_unique_values>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
shop_ids_test = test.shop_id.unique()
for shop_id in [0,1,11,20,8]:
print(shop_id in shop_ids_test )<concatenate>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
test['date_block_num'] = 34
train = pd.concat([train, test.drop('ID', axis=1)], ignore_index=True, sort=False, keys=['date_block_num', 'shop_id', 'item_id'])
 train.fillna(0, inplace=True )<merge>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
index_cols = ['shop_id', 'item_id', 'date_block_num']
grid = []
for block_num in train.date_block_num.unique() :
cur_shops = train[train['date_block_num']==block_num]['shop_id'].unique()
cur_items = train[train['date_block_num']==block_num]['item_id'].unique()
grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])) ,dtype='int32'))
grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)
 train = pd.merge(grid,train,how='left',on=index_cols ).fillna(0)
 train.sort_values(['date_block_num','shop_id','item_id'],inplace=True )<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
cumsum = train.groupby('item_id')['item_cnt_month'].cumsum() - train['item_cnt_month']
cumc = train.groupby('item_id' ).cumcount() + 1
train['item_target_enc'] = cumsum/cumc
train['item_target_enc'].fillna(0.3343, inplace=True)
 encoded_feature = train['item_target_enc'].values<feature_engineering>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
train['month'] = train['date_block_num'].map(lambda month: month-12*(month//12))<data_type_conversions>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
def number_of_weekens_in_month(first_day):
ndays = first_day.daysinmonth
weekends = 0
for i in range(ndays):
if(pd.to_datetime(first_day + timedelta(days=(np.long(i)))).dayofweek in [5, 6]):
weekends = weekends + 1
return weekends<groupby>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
first_days_map = sales_train.groupby('date_block_num')['date'].min().map(lambda date: date.replace(day=1))<categorify>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train['number_of_weekends'] = train['date_block_num'].map(first_days_map.map(number_of_weekens_in_month))<feature_engineering>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
sales_train['month'] = sales_train['date_block_num'].map(lambda month: month-12*(month//12))<feature_engineering>
LR = 0.1
BETA_1 = 0.9
BETA_2 = 0.999
DECAY = 0.01
AMSGRAD = False
train['days_in_month'] = train['date_block_num'].map(first_days_map.map(lambda day: day.daysinmonth))<groupby>
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN-Optimized")
 model.compile(loss=mloss(0.8), optimizer = tf.keras.optimizers.Adam(lr = LR, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = None, decay = DECAY, amsgrad = AMSGRAD), metrics=[score])
 return model
def specify_the_accuracy_group(row):
num_of_block = row.date_block_num - 1
if(num_of_block != -1):
values = train.loc[(train['shop_id']==row.shop_id)&(train['item_id']==row.item_id)&\ 
(train['date_block_num']==num_of_block),['item_cnt_month']].values
return 0 if(values.size==0)else values[0][0]
else:
return 0<merge>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def previous_months_value(df, collumn, offset):
previous_month_values = df.copy()
previous_month_values['date_block_num'] = previous_month_values['date_block_num'] + offset
previous_month_values.rename(columns={collumn: 'prev_month_' + str(offset)+ '_' + collumn}, inplace=True)
 return pd.merge(df, previous_month_values[['date_block_num','shop_id','item_id','prev_month_' + str(offset)+ '_' + collumn]], on=['date_block_num','shop_id','item_id'], how='left' )<define_variables>
NFOLD = 10
kf = KFold(n_splits=NFOLD )
offsets = [1,2,3,9]<define_variables>
%%time
count = 0
for train_idx, val_idx in kf.split(z):
count += 1
print(f"FOLD {count}:")
 net = make_model()
net.fit(z[train_idx], y[train_idx], batch_size=BATCH_SIZE, epochs=850,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("Train:", net.evaluate(z[train_idx], y[train_idx], verbose=0, batch_size=BATCH_SIZE))
print("Val:", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("Predicting Test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
column_to_offset = 'item_cnt_month'<categorify>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
for offset in offsets:
train = previous_months_value(train, column_to_offset, offset)
 values = {'prev_month_' + str(offset)+ '_' + column_to_offset: 0}
train = train.fillna(value=values )<groupby>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
item_price_map = sales_train.loc[sales_train.item_price > 0].groupby(['item_id'])['item_price'].mean()<categorify>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
train['price'] = train['item_id'].map(item_price_map )<merge>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
first_month_product_appeared = sales_train.groupby(['shop_id', 'item_id'])['date_block_num'].min().reset_index()
first_month_product_appeared.rename(columns={'date_block_num': 'first_appeared'}, inplace=True)
 train = pd.merge(train,first_month_product_appeared, on=['shop_id','item_id'], how='left')
 train['first_appeared'] = train['date_block_num'] - train['first_appeared'] + 1
train['first_appeared'] = train['first_appeared'].map(lambda n: 0 if(n < 0)else n)
 values = {'first_appeared': 0}
train = train.fillna(value=values )<groupby>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<categorify><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )