Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<prepare_x_and_y>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, GroupKFold
X_train = test.drop(['item_id','shop_id','2015-10'],axis = 1)
 y_train = test['2015-10']
X_test = test.drop(['item_id','shop_id','2013-01'],axis = 1)
 print('shape of X_train: ',X_train.shape)
 print('shape of y_train: ',y_train.shape)
 print('shape of X_test: ',X_test.shape)
<split>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
X_train,X_val, y_train,y_val = train_test_split(X_train,y_train,test_size = 0.2,random_state=10 )<set_options>
warnings.filterwarnings("ignore")
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 %matplotlib inline
sns.set(style="darkgrid")
 pd.set_option('display.float_format', lambda x: '%.2f' % x)
 warnings.filterwarnings("ignore")
 sys.version_info<load_from_csv>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 items = items.drop('item_name', axis=1)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 test['date_block_num'] = 34 
<categorify>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128
def compress_columns(df,columns,keyword,search_type,datatype):
if search_type=='in':
valid_features = [x for x in columns if keyword in x]
elif search_type=='start':
valid_features = [x for x in columns if x.startswith(keyword)]
if len(valid_features):
for f in valid_features:
df[f] = df[f].round().astype(datatype)
 return df
def data_compression(df):
features = df.columns.values
if 'date_block_num' in features:
df['date_block_num'] = df['date_block_num'].astype(np.int8)
 if 'shop_id' in features:
df['shop_id'] = df['shop_id'].astype(np.int8)
 if 'item_category_id' in features:
df['item_category_id'] = df['item_category_id'].astype(np.int8)
 if 'item_id' in features:
df['item_id'] = df['item_id'].astype(np.int16)
 if 'item_price' in features:
df['item_price'] = df['item_price'].astype(np.float32)
 if 'item_id_avg_item_price' in features:
df['item_id_avg_item_price'] = df['item_id_avg_item_price'].astype(np.float32)
 df = compress_columns(df,features,'item_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'item_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'shop_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'shop_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'shop_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_category_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'item_category_id_sum_item_cnt_day','in',np.int32)
 df = compress_columns(df,features,'item_category_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_cnt_day','start',np.int16)
 return df<categorify>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
train = data_compression(train)
 items = data_compression(items)
 test = data_compression(test )<merge>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
train = pd.merge(train,items,on='item_id',how='left')
 test = pd.merge(test,items, on='item_id', how='left' )<filter>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
train = train[(train['item_price']<100000)&(train['item_price']>=0)]
train = train[(train['item_cnt_day']<1000)&(train['item_cnt_day']>=0)]<feature_engineering>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<merge>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
train_m = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':'sum','item_price':np.mean} ).reset_index()
train_m = pd.merge(grid,train_m,on=['date_block_num','shop_id','item_id'],how='left' ).fillna(0)
 train_m = pd.merge(train_m,items,on='item_id',how='left')
 train_m = data_compression(train_m )<merge>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
for type_id in ['item_id', 'shop_id', 'item_category_id']:
for column_id, aggregator, aggtype in [('item_price',np.mean,'avg'),('item_cnt_day',np.sum,'sum'),('item_cnt_day',np.mean,'avg')]:
mean_df = train.groupby([type_id,'date_block_num'] ).aggregate(aggregator ).reset_index() [[column_id,type_id,'date_block_num']]
mean_df.columns = [type_id+'_'+aggtype+'_'+column_id,type_id,'date_block_num']
train_m = pd.merge(train_m, mean_df, on=['date_block_num',type_id], how='left')
 del mean_df
gc.collect()<set_options>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
del train
gc.collect()<categorify>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )<merge>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
lag_features = list(train_m.columns[6:])+['item_cnt_day']
lags = [1,2,3,6]
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
train_m = pd.merge(train_m,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )<categorify>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )<data_type_conversions>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
max_clip = 30
train_m['item_cnt_day'] = train_m['item_cnt_day'].clip(0,max_clip ).astype(np.float16 )<merge>
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
test = pd.merge(test,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )<categorify>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD
for f in test.columns:
if 'item_cnt' in f:
test[f] = test[f].fillna(0)
 elif 'item_price' in f:
test[f] = test[f].fillna(test[f].median())
 test = data_compression(test )<drop_column>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
cols_to_drop = lag_features[:-1] + ['item_price']
print('Columns to drop')
 print(cols_to_drop )<drop_column>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))
train_cols = train_m.columns.values
test_cols = test.columns.values
for c in cols_to_drop:
if c in train_cols:
train_m = train_m.drop(c,axis=1)
 if c in test_cols:
test = test.drop(c,axis=1 )<data_type_conversions>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
train_m['month'] = train_m['date_block_num']%12
train_m['month'] = train_m['month'].astype(np.int8)
 days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 train_m['days'] = train_m['month'].map(days ).astype(np.int8)
 test['month'] = 11
test['month'] = test['month'].astype(np.int8)
 test['days'] = 30
test['days'] = test['days'].astype(np.int8 )<filter>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 33]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 32]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 31]['item_cnt_day'] )<filter>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
train_m = train_m[train_m['date_block_num']>12]<define_variables>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
train_set = train_m[train_m['date_block_num']<33]
val_set = train_m[train_m['date_block_num']==33]<load_pretrained>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression.csv", index=False )
train_set.to_pickle('train.pkl')
 val_set.to_pickle('val.pkl')
 test.to_pickle('test.pkl' )<set_options>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 submission = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission = submission[['Patient','Weeks','Confidence','Patient_Week']]
submission = submission.merge(chunk.drop('Weeks', axis=1), on="Patient" )
del train_m
gc.collect()<prepare_x_and_y>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
submission['WHERE'] = 'test'
data = tr.append([chunk, submission] )
train_x = train_set.drop(['item_cnt_day'],axis=1)
 train_y = train_set['item_cnt_day']
val_x = val_set.drop(['item_cnt_day'],axis=1)
 val_y = val_set['item_cnt_day']<set_options>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
del train_set
del val_set
gc.collect()<import_modules>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error<train_model>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
gc.collect()
ts = time.time()
lm = linear_model.LinearRegression()
lm.fit(train_x,train_y)
 print('Training time: %s' %(time.time() - ts))<save_to_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
train_pred1 = lm.predict(train_x ).clip(0, 20)
 val_pred1 = lm.predict(val_x ).clip(0, 20)
 test_pred1 = lm.predict(test ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred1))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred1))
print("Val RMSE: %f" %(val_rmse))
sub_df1 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred1})
 sub_df1.to_csv('lm_submission.csv',index=False)
 pickle.dump(train_pred1,open('lm_train.pickle','wb'))
pickle.dump(val_pred1,open('lm_val.pickle','wb'))
pickle.dump(test_pred1,open('lm_test.pickle','wb'))<import_modules>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
submission = data.loc[data.WHERE=='test']
del data
import xgboost as xgb
from xgboost import XGBRegressor
from xgboost import plot_importance<train_model>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="elu", name="d2" )(z)
 x = L.Dense(100, activation="relu", name="d3" )(z)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
gc.collect()
ts = time.time()
xgbtrain = xgb.DMatrix(train_x.values, train_y.values)
 param = {'max_depth':10,
'subsample':1,
'min_child_weight':0.5,
'eta':0.3,
'num_round':1000,
'seed':1,
'verbosity':2,
'eval_metric':'rmse'}
bst = xgb.train(param, xgbtrain)
 print('Training time: %s' %(time.time() - ts))<save_to_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = submission[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
train_pred2 = bst.predict(xgb.DMatrix(train_x.values)).clip(0, 20)
 val_pred2 = bst.predict(xgb.DMatrix(val_x.values)).clip(0, 20)
 test_pred2 = bst.predict(xgb.DMatrix(test.values)).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred2))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred2))
print("Val RMSE: %f" %(val_rmse))
sub_df2 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred2 })
 sub_df2.to_csv('xgb_submission.csv',index=False)
 pickle.dump(train_pred2,open('xgb_train.pickle','wb'))
pickle.dump(val_pred2,open('xgb_val.pickle','wb'))
pickle.dump(test_pred2,open('xgb_test.pickle','wb'))<import_modules>
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']
from sklearn.ensemble import RandomForestRegressor<import_modules>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD
from sklearn.ensemble import RandomForestRegressor<train_model>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
gc.collect()
ts = time.time()
rf = RandomForestRegressor(
 bootstrap=True,
max_depth=100,
max_features= 3,
min_samples_leaf=5,
min_samples_split=12,
n_estimators=100,
random_state=42,
verbose=1,
n_jobs=-1 
)
 rf.fit(train_x.values,train_y.values)
 print('Training time: %s' %(time.time() - ts))<save_to_csv>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))
train_pred3 = rf.predict(train_x ).clip(0, 20)
 val_pred3 = rf.predict(val_x ).clip(0, 20)
 test_pred3 = rf.predict(test ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred3))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred3))
print("Val RMSE: %f" %(val_rmse))
sub_df3 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred3})
 sub_df3.to_csv('rf_submission.csv',index=False)
 pickle.dump(train_pred3,open('rf_train.pickle','wb'))
pickle.dump(val_pred3,open('rf_val.pickle','wb'))
pickle.dump(test_pred3,open('rf_test.pickle','wb'))<create_dataframe>
submission['FVC1'] = pe[:, 1]
submission['Confidence1'] = pe[:, 2] - pe[:, 0]
train_stack = pd.DataFrame({'lm':train_pred1,'xgboost':train_pred2,'rf':train_pred3})
 val_stack = pd.DataFrame({'lm':val_pred1,'xgboost':val_pred2,'rf':val_pred3})
 test_stack = pd.DataFrame({'lm':test_pred1,'xgboost':test_pred2,'rf':test_pred3} )<train_model>
submission1 = submission[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
gc.collect()
ts = time.time()
stack_lm = linear_model.LinearRegression()
stack_lm.fit(train_stack,train_y)
 print('Training time: %s' %(time.time() - ts))<save_to_csv>
submission1.loc[~submission1.FVC1.isnull() ,'FVC'] = submission1.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
submission1['Confidence'] = sigma_opt
else:
submission1.loc[~submission1.FVC1.isnull() ,'Confidence'] = submission1.loc[~submission1.FVC1.isnull() ,'Confidence1']
train_pred4 = stack_lm.predict(train_stack ).clip(0, 20)
 val_pred4 = stack_lm.predict(val_stack ).clip(0, 20)
 test_pred4 = stack_lm.predict(test_stack ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred4))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred4))
print("Val RMSE: %f" %(val_rmse))
sub_df4 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred4})
 sub_df4.to_csv('stack_submission.csv',index=False)
 pickle.dump(train_pred4,open('stack_train.pickle','wb'))
pickle.dump(val_pred4,open('stack_val.pickle','wb'))
pickle.dump(test_pred4,open('stack_test.pickle','wb'))<choose_model_class>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<save_to_csv>
submission1[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression_V2.csv", index=False )
<set_options>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 submission2 = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission2['Patient'] = submission2['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission2['Weeks'] = submission2['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission2 = submission2[['Patient','Weeks','Confidence','Patient_Week']]
submission2 = submission2.merge(test.drop('Weeks', axis=1), on="Patient" )
env = riiideducation.make_env()<load_from_csv>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
submission2['WHERE'] = 'test'
data = train.append([test, submission2] )
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)<sort_values>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<count_unique_values>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
train[(train.content_type_id == False)].task_container_id.nunique()<groupby>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2<groupby>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
group3['avg_questions_seen'] = group3.avg_questions.cumsum()<set_options>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
submission2 = data.loc[data.WHERE=='test']
del data
group3.iloc[0].avg_questions_seen<groupby>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.11, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']<merge>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )<groupby>
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']<groupby>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']<merge>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<merge>
submission2['FVC1'] = pe[:, 1]
submission2['Confidence1'] = pe[:, 2] - pe[:, 0]
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<feature_engineering>
submission2 = submission2[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
question2.quest_pct = round(question2.quest_pct,5 )<drop_column>
submission2.loc[~submission2.FVC1.isnull() ,'FVC'] = submission2.loc[~submission2.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
submission2['Confidence'] = sigma_opt
else:
submission2.loc[~submission2.FVC1.isnull() ,'Confidence'] = submission2.loc[~submission2.FVC1.isnull() ,'Confidence1']
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )<filter>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
submission2.loc[submission2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission2.loc[submission2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )<groupby>
submission2[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression_V3.csv", index=False)
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 submission3 = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission3['Patient'] = submission3['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission3['Weeks'] = submission3['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission3 = submission3[['Patient','Weeks','Confidence','Patient_Week']]
submission3 = submission3.merge(chunk.drop('Weeks', axis=1), on="Patient" )
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )<groupby>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
submission3['WHERE'] = 'test'
data = tr.append([chunk, submission3] )
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
del(train )<merge>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left" )<merge>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
submission3 = data.loc[data.WHERE=='test']
del data
tr.shape, chunk.shape, submission3.shape
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left" )<categorify>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(200, activation="relu", name="d1" )(z)
 x = L.Dense(200, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<load_from_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
<categorify>
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1<prepare_x_and_y>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
submission3['FVC1'] = pe[:, 1]
submission3['Confidence1'] = pe[:, 2] - pe[:, 0]
submission3 = submission3[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]<data_type_conversions>
submission3.loc[~submission3.FVC1.isnull() ,'FVC'] = submission3.loc[~submission3.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
submission3['Confidence'] = sigma_opt
else:
submission3.loc[~submission3.FVC1.isnull() ,'Confidence'] = submission3.loc[~submission3.FVC1.isnull() ,'Confidence1']
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
submission3.loc[submission3['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission3.loc[submission3['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
submission3[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression_v4.csv", index=False )
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
<data_type_conversions>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 submission4 = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission4['Patient'] = submission4['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission4['Weeks'] = submission4['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission4 = submission4[['Patient','Weeks','Confidence','Patient_Week']]
submission4 = submission4.merge(chunk.drop('Weeks', axis=1), on="Patient" )
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )<create_dataframe>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
submission4['WHERE'] = 'test'
data = tr.append([chunk, submission4] )
params = {
'objective': 'binary',
'boosting' : 'gbdt',
'max_bin': 800,
'learning_rate': 0.0175,
'num_leaves': 80
}
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'])
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train )<train_model>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
submission4 = data.loc[data.WHERE=='test']
del data
tr.shape, chunk.shape, sub.shape 
(( 1535, 22),(5, 22),(730, 22))
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=1300,
early_stopping_rounds=8
)<compute_train_metric>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']
y_pred = model.predict(X_val)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )<import_modules>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD
import matplotlib.pyplot as plt
import seaborn as sns<split>
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))
iter_test = env.iter_test()<merge>
submission4['FVC1'] = pe[:, 1]
submission4['Confidence1'] = pe[:, 2] - pe[:, 0]
submission4 = submission4[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
submission4.loc[~submission4.FVC1.isnull() ,'FVC'] = submission4.loc[~submission4.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
submission4['Confidence'] = sigma_opt
else:
submission4.loc[~submission4.FVC1.isnull() ,'Confidence'] = submission4.loc[~submission4.FVC1.isnull() ,'Confidence1']
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
submission4.loc[submission4['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission4.loc[submission4['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
sns.set()
py.init_notebook_mode(connected=True)
 os.listdir(".. /input/" )<define_variables>
submission4[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression_v5.csv", index=False )
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"<load_from_csv>
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print('Train shape: ', df_train.shape)
 print('Number of unique customers in train: {}'.format(df_train['Patient'].nunique()))
print('Test shape:', df_test.shape )
train_info = pd.read_csv(basepath + "train.csv")
 train_info.head()<load_from_csv>
df_base = df_train.drop_duplicates(subset='Patient', keep='first')
 df_base = df_base[['Patient', 'Weeks', 'FVC',
'Percent', 'Age']].rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_base.head(3 )
test_info = pd.read_csv(basepath + "test.csv")
 test_info.head()<sort_values>
df_train['visit'] = 1
df_train['visit'] = df_train[['Patient', 'visit']].groupby('Patient' ).cumsum()
df_train = df_train.loc[df_train['visit'] > 0, :]
missing_vals_train = train_info.isnull().sum() / train_info.shape[0]
missing_vals_train[missing_vals_train > 0].sort_values(ascending=False )<sort_values>
df_train = pd.merge(df_train,
df_base,
on='Patient',
how='left')
 print(df_train.shape)
 df_train.head(3 )
missing_vals_test = test_info.isnull().sum() / test_info.shape[0]
missing_vals_test[missing_vals_test > 0].sort_values(ascending=False )<count_values>
df_train['weeks_passed'] = df_train['Weeks'] - df_train['base_week']
df_train = pd.get_dummies(df_train, columns=['Sex', 'SmokingStatus'])
 sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 sub.head()
train_info.image_name.value_counts().max()<count_values>
df_test = df_test.rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_test = pd.merge(sub,
df_test,
on='Patient',
how='right')
 df_test = pd.get_dummies(df_test, columns=['Sex', 'SmokingStatus'])
 df_test['weeks_passed'] = df_test['Weeks'] - df_test['base_week']
df_test.head()
test_info.image_name.value_counts().max()<count_values>
missing_columns = np.setdiff1d(df_train.drop(['Patient', 'FVC', 'Percent', 'Age', 'visit'], axis = 1 ).columns, df_test.columns)
 if len(missing_columns)> 0:
print('/!\ Missing columns in test: ', missing_columns)
 for col in missing_columns:
df_test[col] = 0
train_info.patient_id.value_counts().max()<count_values>
def OSIC_metric(y_true, y_pred, y_pred_std):
delta = np.clip(abs(y_true - y_pred), 0, 1000)
 std_clipped = np.clip(y_pred_std, 70, np.inf)
 return np.mean(-(np.sqrt(2)*delta/std_clipped)- np.log(np.sqrt(2)*std_clipped))
test_info.patient_id.value_counts().max()<feature_engineering>
class Model() :
def __init__(self, model=BayesianRidge(alpha_1=0.1,alpha_2=0.1,lambda_1=0.03,lambda_2=0.01), n_splits=2):
self.regressor = model
self.n_splits = n_splits
self.gkf = GroupKFold(n_splits=n_splits)
 self.train_cols = ['Weeks', 'base_week', 'base_FVC',
'base_percent', 'base_age', 'weeks_passed', 'Sex_Female',
'Sex_Male', 'SmokingStatus_Currently smokes',
'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']
def fit(self, X, y):
self.regressor.fit(X, y)
 def predict(self, X):
pred = self.regressor.predict(X, return_std=True)
 return pred
def fit_predict_cv(self, df, df_test=pd.DataFrame()):
scores = np.zeros(( self.n_splits,))
oof = np.zeros(( len(df),))
oof_std = np.zeros_like(oof)
 if len(df_test)> 0:
pred_sub = np.zeros(( len(df_test), self.n_splits))
pred_sub_std = np.zeros_like(pred_sub)
 target = 'FVC'
for i,(train_idx, val_idx)in enumerate(self.gkf.split(df, groups=df['Patient'])) :
X_train = df.loc[train_idx, self.train_cols]
y_train = df.loc[train_idx, target]
X_val = df.loc[val_idx, self.train_cols]
y_val = df.loc[val_idx, target]
self.fit(X_train, y_train)
 pred_train, pred_train_std = self.predict(X_train)
 pred_val, pred_val_std = self.predict(X_val)
 if len(df_test)> 0:
pred_sub[:, i], pred_sub_std[:, i] = self.predict(df_test[self.train_cols])
 oof[val_idx] = pred_val
oof_std[val_idx] = pred_val_std
print('Train score: {0:.2f} | Test score: {1:.2f}'.format(OSIC_metric(y_train, pred_train, pred_train_std),
OSIC_metric(y_val, pred_val, pred_val_std)))
 print('OOF score: {0:.4f}'.format(OSIC_metric(df[target], oof, oof_std)))
 res = dict()
res['oof'] = oof
res['oof_std'] = oof_std
if len(df_test)> 0:
res['pred_sub'] = pred_sub.mean(axis=1)
 res['pred_sub_std'] = pred_sub_std.mean(axis=1)
 return res
fvc_model = Model()
res = fvc_model.fit_predict_cv(df_train, df_test)
200/test_info.shape[0] * 100<count_unique_values>
df_test['FVC'] = res['pred_sub']
df_test['Confidence'] = res['pred_sub_std']
submission5 = sub[['Patient_Week']]
submission5 = pd.merge(submission5,
df_test[['Patient_Week', 'FVC', 'Confidence']],
on='Patient_Week',
how='left')
 submission5.head()
train_info.groupby("benign_malignant" ).target.nunique()<groupby>
warnings.filterwarnings("ignore" )
patient_ages_table_train = train_info.groupby(["patient_id", "age_approx"] ).size() / train_info.groupby("patient_id" ).size()
patient_ages_table_train = patient_ages_table_train.unstack().transpose()
patient_ages_table_test = test_info.groupby(["patient_id", "age_approx"] ).size() / test_info.groupby("patient_id" ).size()
patient_ages_table_test = patient_ages_table_test.unstack().transpose()
patient_with_known_ages_train = train_info[train_info.patient_id.isin(patient_ages_table_train.columns.values)]
sorted_patients_train = patient_with_known_ages_train.patient_id.value_counts().index.values
patient_with_known_ages_test = test_info[test_info.patient_id.isin(patient_ages_table_test.columns.values)]
sorted_patients_test = patient_with_known_ages_test.patient_id.value_counts().index.values
fig, ax = plt.subplots(2,1, figsize=(20,20))
sns.heatmap(patient_ages_table_train[sorted_patients_train], cmap="Reds", ax=ax[0], cbar=False);
ax[0].set_title("Image coverage in % per patient and age in train data");
sns.heatmap(patient_ages_table_test[sorted_patients_test], cmap="Blues", ax=ax[1], cbar=False);
ax[1].set_title("Image coverage in % per patient and age in test data");
ax[0].set_xlabel("")
 ax[1].set_xlabel("");<feature_engineering>
def seed_everything(seed=777):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
patient_gender_train = train_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 patient_gender_test = test_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 train_patients = pd.DataFrame(index=patient_gender_train.index.values, data=patient_gender_train.values, columns=["sex"])
 test_patients = pd.DataFrame(index=patient_gender_test.index.values, data=patient_gender_test.values, columns=["sex"])
 train_patients.loc[:, "num_images"] = train_info.groupby("patient_id" ).size()
test_patients.loc[:, "num_images"] = test_info.groupby("patient_id" ).size()
train_patients.loc[:, "min_age"] = train_info.groupby("patient_id" ).age_approx.min()
train_patients.loc[:, "max_age"] = train_info.groupby("patient_id" ).age_approx.max()
test_patients.loc[:, "min_age"] = test_info.groupby("patient_id" ).age_approx.min()
test_patients.loc[:, "max_age"] = test_info.groupby("patient_id" ).age_approx.max()
train_patients.loc[:, "age_span"] = train_patients["max_age"] - train_patients["min_age"]
test_patients.loc[:, "age_span"] = test_patients["max_age"] - test_patients["min_age"]
train_patients.loc[:, "benign_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "benign"]
train_patients.loc[:, "malignant_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "malignant"]
train_patients["min_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.min().loc[:, "malignant"]
train_patients["max_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.max().loc[:, "malignant"]<sort_values>
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 37
seed_everything(seed=SEED)
 N_FOLD = 5
train_patients.sort_values(by="malignant_cases", ascending=False ).head()<define_variables>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
example_files = os.listdir(basepath + "train/")[0:2]
example_files<feature_engineering>
train = pd.concat([train,otest])
 output = pd.DataFrame()
gb = train.groupby('Patient')
 tk0 = tqdm(gb, total=len(gb))
for _, usr_df in tk0:
usr_output = pd.DataFrame()
for week, tmp in usr_df.groupby('Weeks'):
rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}
tmp = tmp.rename(columns=rename_cols)
 drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']
_usr_output = usr_df.drop(columns=drop_cols ).rename(columns={'Weeks': 'predict_Week'} ).merge(tmp, on='Patient')
 _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']
usr_output = pd.concat([usr_output, _usr_output])
 output = pd.concat([output, usr_output])
 train = output[output['Week_passed']!=0].reset_index(drop=True )
train_info["dcm_path"] = basepath + "train/" + train_info.image_name + ".dcm"
test_info["dcm_path"] = basepath + "test/" + test_info.image_name + ".dcm"<load_from_csv>
test = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})
 submission6 = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 submission6['Patient'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[0])
 submission6['predict_Week'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 test = submission6.drop(columns=['FVC', 'Confidence'] ).merge(test, on='Patient')
 test['Week_passed'] = test['predict_Week'] - test['base_Week']
test.set_index('Patient_Week', inplace=True )
example_dcm = pydicom.dcmread(train_info.dcm_path[2])
 example_dcm<feature_engineering>
folds = train[['Patient', TARGET]].copy()
Fold = GroupKFold(n_splits=N_FOLD)
 groups = folds['Patient'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[TARGET], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int )
train_info["image_path"] = basepath + "jpeg/train/" + train_info.image_name + ".jpg"
test_info["image_path"] = basepath + "jpeg/test/" + test_info.image_name + ".jpg"<load_from_csv>
def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):
trn_idx = folds[folds.fold!=fold_num].index
val_idx = folds[folds.fold==fold_num].index
y_tr = target.iloc[trn_idx].values
X_tr = train_df.iloc[trn_idx][features].values
y_val = target.iloc[val_idx].values
X_val = train_df.iloc[val_idx][features].values
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
clf.fit(X_tr, y_tr)
 oof[val_idx] = clf.predict(X_val)
 predictions += clf.predict(test_df[features])
 return oof, predictions
def run_kfold_model(clf, train, test, folds, features, target, n_fold=5):
oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_ in range(n_fold):
_oof, _predictions = run_single_model(clf,
train,
test,
folds,
features,
target,
fold_num=fold_)
 oof += _oof
predictions += _predictions/n_fold
return oof, predictions
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )<load_from_csv>
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']
features = [c for c in features if c not in drop_features]
if cat_features:
ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')
 ce_oe.fit(train)
 train = ce_oe.transform(train)
 test = ce_oe.transform(test )
train_image_stats_1 = pd.read_csv(imagestatspath + "train_image_stats_10000.csv")
 train_image_stats_2 = pd.read_csv(imagestatspath + "train_image_stats_20000.csv")
 train_image_stats_3 = pd.read_csv(imagestatspath + "train_image_stats_toend.csv")
 train_image_stats_4 = train_image_stats_1.append(train_image_stats_2)
 train_image_stats = train_image_stats_4.append(train_image_stats_3)
 train_image_stats.shape<define_variables>
clf = Ridge(alpha=0.1)
 oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)
 train['FVC_pred'] = oof
test['FVC_pred'] = predictions
plot_test = True<groupby>
train['Confidence'] = 100
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
test_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / test_image_stats.shape[0]<groupby>
def loss_func(weight, row):
confidence = weight
sigma_clipped = max(confidence, 70)
 diff = abs(row['FVC'] - row['FVC_pred'])
 delta = min(diff, 1000)
 score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)
 return -score
results = []
tk0 = tqdm(train.iterrows() , total=len(train))
for _, row in tk0:
loss_partial = partial(loss_func, row=row)
 weight = [100]
result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')
 x = result['x']
results.append(x[0] )
train_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / train_image_stats.shape[0]<define_variables>
train['Confidence'] = results
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
examples1 = {"rows": 1080, "columns": 1920}
examples2 = {"rows": 4000, "columns": 6000}<load_from_csv>
TARGET = 'Confidence'
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']
features = [c for c in features if c not in drop_features]
oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD )
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
train['Confidence'] = oof
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
!pip install -q efficientnet >> /dev/null<import_modules>
test['Confidence'] = predictions
test = test.reset_index()
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_variables>
submission6 = submission6[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')
 submission6 = submission6.rename(columns={'FVC_pred': 'FVC'} )
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
for i in range(len(otest)) :
submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
submission6.to_csv('submission_Ridge.csv', index=False, float_format='%.1f')
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
submission_final =(subm.drop(['Patient_Week'], axis=1)+ submission1.drop(['Patient_Week'], axis=1)+ submission2.drop(['Patient_Week'], axis=1)+ submission3.drop(['Patient_Week'], axis=1)+ submission4.drop(['Patient_Week'], axis=1)) /5
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
submission_final['Patient_Week'] = subm['Patient_Week']
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
submission_final.drop(['FVC1','Confidence1'],axis=1,inplace=True )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
submission_final=pd.merge(submission_final,submission5,on='Patient_Week',how='inner' ).merge(submission6,on='Patient_Week',how='inner' ).rename(columns={'FVC':'FVC1','Confidence':'Confidence1'} )
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
submission_final['FVC']=(submission_final['FVC_x']+submission_final['FVC_y']+submission_final['FVC1'])/3
submission_final['Confidence'] =(submission_final['Confidence1']+submission_final['Confidence_x']+submission_final['Confidence_y'])/3
<init_hyperparams><EOS> 
submission_final[['Patient_Week','FVC','Confidence']].to_csv("submission.csv",index=False )