Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<drop_column>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<data_type_conversions>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )<data_type_conversions>
seed_everything(42 )
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )<categorify>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )<data_type_conversions>
Dropout_model = 0.3855
FVC_weight = 0.2
Confidence_weight = 0.2
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<data_type_conversions>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )<count_missing_values>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
all_data.isna().sum()<feature_engineering>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
all_data["AllSF"] = all_data["GrLivArea"] + all_data["TotalBsmtSF"]
all_data['Area'] = all_data['LotArea']*all_data['LotFrontage']
all_data['Area_log'] = np.log1p(all_data['Area'] )<define_variables>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
num_cols = all_data.columns[all_data.dtypes != 'object']<count_unique_values>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
c = []
for col in num_cols:
if(all_data[col].nunique() < 15):
c.append(col)
<drop_column>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
c.remove('PoolArea' )<data_type_conversions>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
for col in c:
all_data[col] = all_data[col].astype(str)
<define_variables>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
cats = all_data.columns[all_data.dtypes == 'object']<define_variables>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
ordinals = ['MSSubClass','ExterQual','LotShape','BsmtQual','BsmtCond','BsmtExposure',
'BsmtFinType1', 'BsmtFinType2','HeatingQC','Functional','FireplaceQu','KitchenQual',
'GarageFinish','GarageQual','GarageCond','PoolQC','Fence']
orders=[ ['20','30','40','45','50','60','70','75','80','85', '90','120','150','160','180','190'],
['Fa','TA','Gd','Ex'], ['Reg','IR1' ,'IR2','IR3'],
['None','Fa','TA','Gd','Ex'], ['None','Po','Fa','TA','Gd','Ex'],
['None','No','Mn','Av','Gd'],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['Po','Fa','TA','Gd','Ex'], ['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],
['None','Po','Fa','TA','Gd','Ex'], ['Fa','TA','Gd','Ex'],
['None','Unf','RFn','Fin'], ['None','Po','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Fa','Gd','Ex'], ['None','MnWw','GdWo','MnPrv','GdPrv'] ]<import_modules>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
from sklearn.preprocessing import OrdinalEncoder<categorify>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
for i in range(len(orders)) :
oe = OrdinalEncoder(categories={0:orders[i]})
 all_data.loc[:,ordinals[i]] = oe.fit_transform(all_data.loc[:,ordinals[i]].values.reshape(-1,1))<feature_engineering>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<data_type_conversions>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
for i in ordinals:
all_data[i] = all_data[i].astype('str' )<create_dataframe>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
nums = all_data.columns[all_data.dtypes != 'object']
skews = all_data[nums].apply(lambda x : skew(x.dropna())).sort_values(ascending = False)
 skewness = pd.DataFrame({'Skewness':skews} )<filter>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
skewness = skewness[abs(skewness)>0.75]<import_modules>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
from scipy.special import boxcox1p<define_variables>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
skewed_features = skewness.index<feature_engineering>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
skewed_features
lam = 0.15
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam )<concatenate>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
rest = set(all_data.columns ).difference(ordinals)
 len(rest )<categorify>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
OHE_encoded = pd.get_dummies(all_data[rest],drop_first=True )<concatenate>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
df = pd.concat(( OHE_encoded, all_data[ordinals]), axis = 1 )<data_type_conversions>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
for col in df:
df[col] = pd.to_numeric(df[col] )<split>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
train = df[:n_train]
test = df[n_train:]<import_modules>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
from sklearn.svm import SVR<install_modules>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
!pip install pycaret[full] 
<split>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
X_train, X_val, y_train, y_val = train_test_split(train, target, test_size =.2 )<compute_train_metric>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
def rmse_cv(model):
kf = KFold(n_splits = 6, shuffle = True, random_state = 42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, target, scoring ='neg_mean_squared_error', cv = kf))
rmse_val = np.sqrt(-cross_val_score(model, train.values, target, scoring = 'neg_mean_squared_error', cv = kf))
return "Train ",rmse, "Validation", rmse_val<choose_model_class>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0002144127223451149, max_iter=200000))<choose_model_class>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0010936129372972772, l1_ratio=0.21322431030386174, random_state=3, max_iter= 200000))<choose_model_class>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
KRR = KernelRidge(alpha=0.015, kernel='polynomial', degree=1, coef0=0.5)
<choose_model_class>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
GBoost = GradientBoostingRegressor(n_estimators=1070, max_depth = 3, min_samples_split = 0.9900745088899041,
min_samples_leaf = 0.0024212450940811515, max_features = 'log2',
learning_rate = 0.23500000000000001, loss = 'huber',
criterion = 'mse')
<choose_model_class>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
model_xgb = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.9, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.191, max_delta_step=0, max_depth=2,
min_child_weight=3, monotone_constraints='() ',
n_estimators=180, n_jobs=-1, num_parallel_tree=1,
objective='reg:squarederror', random_state=123, reg_alpha=0.001,
reg_lambda=0.4, scale_pos_weight=13.8, subsample=0.7,
tree_method='auto', validate_parameters=1, verbosity=0 )<choose_model_class>
df.to_csv('submission.csv', index=False )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<find_best_params>
df_issa1 = df.copy()
model_svr = SVR()
model_svr.get_params()<compute_test_metric>
Dropout_model = 0.385
FVC_weight = 0.2
Confidence_weight = 0
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
score = rmse_cv(lasso)
 print("
Lasso score on Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("
Lasso score on Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std()))<compute_test_metric>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)
 train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
score = rmse_cv(ENet)
 print("
ElasticNet score on Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("
ElasticNet score on Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<compute_test_metric>
def expandFVC_Percent(DS):
DS['FVC_Percent'] = DS['FVC'] / DS['Percent']
return DS
score = rmse_cv(KRR)
 print("
Kernel Ridge score on Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("
Kernel Ridge score Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<compute_test_metric>
def expandFE(data_set):
data_set['MinWeeks'] = data_set.groupby('Patient')['Weeks'].transform('min')
 sub_tr = data_set.loc[data_set['Weeks']==data_set['MinWeeks']]
sub_tr['MinFVC'] = sub_tr['FVC']
sub_tr = sub_tr[['Patient', 'MinFVC']]
merged = data_set.merge(sub_tr, on='Patient', how='outer')
 data_set=merged
data_set['MinFVC'] =(data_set['MinFVC'] - data_set['MinFVC'].min())/(data_set['MinFVC'].max() -data_set['MinFVC'].min())
 print(data_set.sample(5))
return data_set
score = rmse_cv(GBoost)
 print("
Gradient Boosting scoreon Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("
Gradient Boosting score Test aSet: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<compute_train_metric>
train = expandFE(train)
score = rmse_cv(model_xgb)
 print("
Xgboost Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("
Xgboost Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<compute_train_metric>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=8):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return math.ceil(len(self.keys)/ float(self.batch_size))
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
self.train_data[k].sort()
mx_len = len(self.train_data[k])
 i = np.random.choice(self.train_data[k][mx_len//3:mx_len//3*2], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x, a, tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
score = rmse_cv(model_lgb)
 print("\LGM Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("\LGM Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std()))<import_modules>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(5,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 return model
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, KFold
from sklearn import metrics<import_modules>
efn_path = ".. /input/trainefn/efn.ckpt"
model.load_weights(efn_path )
from skopt import space
from skopt import gp_minimize
from sklearn.metrics import mean_squared_error<import_modules>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 subs = []
q = -1
from hyperopt import hp, fmin,tpe, Trials
from hyperopt.pyll import scope 
<set_options>
q = -1
with open('.. /input/trainefn/q.txt', 'r')as f:
q=float(f.readlines() [0].strip())
 print(q )
restore_these_settings = np.geterr()
temp_settings = restore_these_settings.copy()
temp_settings["over"] = "ignore"
temp_settings["under"] = "ignore"
np.seterr(**temp_settings)
<init_hyperparams>
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
param_space = {
'loss' : hp.choice('loss', ['ls', 'lad', 'huber', 'quantile']),
'learning_rate' : scope.float(hp.quniform('learning_rate',0.1,1, 0.005)) ,
'n_estimators' : scope.int(hp.quniform('n_estimators', 20,4000,10)) ,
'criterion' : hp.choice('criterion', ['friedman_mse','mse']),
'min_samples_split' : hp.uniform('min_samples_split', 0.0,1.0,),
'min_samples_leaf' : scope.float(hp.uniform('min_samples_leaf', 0.0,0.5)) ,
'max_depth' : scope.int(hp.quniform('max_depth', 2,20,1)) ,
'max_features' : hp.choice('max_features',['auto', 'sqrt', 'log2',])
 } 
<train_model>
N = len(subs)
 print(N)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N)
 sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)
 img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
print(img_sub.sample(5))
def optimize(params, x, y):
model = GradientBoostingRegressor(**params)
 kf = KFold(n_splits=5)
 accuracies = []
for train_idx, test_idx in kf.split(X=x):
xtrain = x[train_idx]
ytrain = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
model.fit(xtrain, ytrain)
 preds = model.predict(x_test)
 fold_acc = mean_squared_error(preds, y_test)
 accuracies.append(fold_acc)
 return np.mean(accuracies)
<install_modules>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
!pip install scikit-opt 
<train_model>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 data['FVC_Percent'] = data['FVC'] / data['Percent']
FE += ['age','percent','week','BASE']
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
print(FE )
optimization_function = partial(
 optimize,
x = train.values,
y = target)
 trials = Trials()<find_best_params>
y = tr['FVC'].values
z = tr[FE].values
print(z.shape)
 ze = sub[FE].values
print(ze.shape)
 nh = z.shape[1]
print(nh)
 pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
net = make_model(nh)
 print(net.summary())
 print(net.count_params())
 NFOLD = 5
kf = KFold(n_splits=NFOLD )
results = fmin(fn = optimization_function,
space = param_space,
max_evals = 100,
trials=trials,
algo = tpe.suggest,
verbose = 10)
 print(results)
<predict_on_test>
cnt = 0
EPOCHS = 800
y = y.astype(np.float32)
 for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
df_issa2 = df.copy()
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmse_cv(averaged_models)
 print("\Averaged base models score Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("\Averaged base models score Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<train_model>
df_issa1.to_csv('df_issa1.csv', index=False)
 df_issa2.to_csv('df_issa2.csv', index=False )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
x = 0.8
df = df_issa1.copy()
df['FVC'] = x*df_issa1['FVC'] +(1-x)*df_issa2['FVC']
df['Confidence'] = x*df_issa1['Confidence'] +(1-x)*df_issa2['Confidence']
df.head()
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmse_cv(stacked_averaged_models)
 print("\Stacking base models score Training Set: {:.4f}({:.4f})
 ".format(score[1].mean() , score[1].std()))
print("\Stacking base models score Test Set: {:.4f}({:.4f})
 ".format(score[3].mean() , score[3].std())) 
<compute_test_metric>
df.to_csv('submission.csv', index=False )