Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<merge>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
warnings.filterwarnings("ignore")
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
print('Training Data:',train.info() , end = "
")
 print('Testing Data:',test.info() )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
print(train.shape, test.shape, sub.shape, data.shape)
 print(train.Patient.nunique() , test.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
matrix['month'] = matrix['date_block_num'] % 12<categorify>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8375), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
data = pd.read_pickle('data.pkl' )<drop_column>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
X = data[data.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 Y = data[data.date_block_num <= 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<train_model><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )