Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<sort_values>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
top_country_f = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['Fatalities'], ascending=False)
 top_country_f.head(20 )<feature_engineering>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
train['MortalityRate'] = train['Fatalities'] / train['ConfirmedCases']
train['MortalityRate'] = train['MortalityRate'].fillna(0.0 )<sort_values>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
top_country_m = train[train['Date'] == curr_date].groupby(['Country_Region'] ).sum().sort_values(['MortalityRate'], ascending=False)
 top_country_m.head(10 )<data_type_conversions>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
for df in [test]:
df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True)
 df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True )<drop_column>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
missed = "NA"
def State(state, country):
if state == missed: return country
return state<data_type_conversions>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
for df in [train, test]:
df['Province_State'].fillna(missed, inplace=True)
 df['Province_State'] = df.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : State(x['Province_State'], x['Country_Region']), axis=1)
 df.loc[:, 'Date'] = df.Date.dt.strftime("%m%d")
 df["Date"] = df["Date"].astype(int )<categorify>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
label_encoder = preprocessing.LabelEncoder()
for df in [train, test]:
df['Country_Region'] = label_encoder.fit_transform(df['Country_Region'])
 df['Province_State'] = label_encoder.fit_transform(df['Province_State'] )<choose_model_class>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
def RF() :
model = RandomForestRegressor(n_estimators = 100)
 return model
def XGB() :
model = XGBRegressor(n_estimators=1700, learning_rate=0.6,
n_job=5)
 return model
def LGBM() :
model = LGBMRegressor(iterations=2)
 return model<data_type_conversions>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in range(len(unique_countries)) :
current_country_train = train.loc[train['Country_Region'] == unique_countries[country]]
current_country_test = test.loc[test['Country_Region'] == unique_countries[country]]
features = ['Country_Region', 'Province_State', 'Date']
X_train = current_country_train[features].to_numpy()
y1_train = current_country_train[['ConfirmedCases']].to_numpy()
y2_train = current_country_train[['Fatalities']].to_numpy()
X_test = current_country_test[features].to_numpy()
y1_train = y1_train.reshape(-1)
 y2_train = y2_train.reshape(-1)
 model1 = XGB()
model1.fit(X_train, y1_train)
 res_cnf_cls = np.round(model1.predict(X_test))
model2 = XGB()
model2.fit(X_train, y2_train)
 res_fac = np.round(model2.predict(X_test))
current_country_test_Id = current_country_test.loc[:, 'ForecastId']
pred = pd.DataFrame({'ForecastId': current_country_test_Id, 'ConfirmedCases': res_cnf_cls, 'Fatalities': res_fac})
 sub = pd.concat([sub, pred], axis=0 )<save_to_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
sub.ForecastId = sub.ForecastId.astype('int')
 sub.to_csv('submission.csv', index=False)
 sub<import_modules>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from datetime import datetime
from sklearn import preprocessing
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<prepare_x_and_y>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
xtrain=train[['Id']]
test['Id']=test['ForecastId']
xtest=test[['Id']]
ytrain_cc=train[['ConfirmedCases']]
ytrain_ft=train[['Fatalities']]<split>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="relu", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005, amsgrad=False), metrics=[score])
 return model
xtr=np.array_split(xtrain,313)
 ycc=np.array_split(ytrain_cc,313)
 yft=np.array_split(ytrain_ft,313)
 xte=np.array_split(xtest,313 )<define_variables>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
a=np.max(xtr[0] ).values
b=a-71
b=b[0]<feature_engineering>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
xte[0]=xte[0]+a
for i in range(312):
xte[i+1]=xte[0]
for i in range(312):
xtr[i+1]=xtr[0]<predict_on_test>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
poly=PolynomialFeatures(3)
 y_pred_cc=[]
for i in range(313):
xtr[i]=poly.fit_transform(xtr[i])
 xte[i]=poly.fit_transform(xte[i])
 model=XGBRegressor()
model.fit(xtr[i],ycc[i]);
y_pr_cc=model.predict(xte[i])
 ycc[i]= ycc[i][71:]
y_pr_cc=y_pr_cc[b:]
y_pr_cc=np.append(ycc[i], y_pr_cc)
 y_pred_cc.append(y_pr_cc )<predict_on_test>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
y_pred_ft=[]
for i in range(313):
model=XGBRegressor()
model.fit(xtr[i],yft[i]);
y_pr_ft=model.predict(xte[i])
 yft[i]= yft[i][71:]
y_pr_ft=y_pr_ft[b:]
y_pr_ft=np.append(yft[i], y_pr_ft)
 y_pred_ft.append(y_pr_ft )<define_variables>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
y_pred_ft_1 = [item for sublist in y_pred_ft for item in sublist]
y_pred_cc_1 = [item for sublist in y_pred_cc for item in sublist]<save_to_csv>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
df_final=pd.DataFrame({'ForecastId':submission.ForecastId, 'ConfirmedCases':y_pred_cc_1, 'Fatalities':y_pred_ft_1})
 df_final.to_csv('/kaggle/working/submission.csv', index=False)
<load_from_csv>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
data=pd.read_csv('/kaggle/working/submission.csv')
 data.head(20 )<load_from_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<categorify><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )