Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<feature_engineering>
import numpy as np
import random
import pandas as pd
import pydicom
import os
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
df.loc[(df.MSZoning.isnull())&(df.Neighborhood == "IDOTRR"), "MSZoning"] = "C(all)"
df.loc[(df.MSZoning.isnull())&(df.Neighborhood == "Mitchel"), "MSZoning"] = "RL"<count_values>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
df.KitchenQual.value_counts()<count_values>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
print(df.groupby(["OverallQual", "KitchenAbvGr"] ).KitchenQual.value_counts())
 df.loc[(df.KitchenQual.isnull())&(df.OverallQual == 5)&(df.KitchenAbvGr == 1), "KitchenQual"] = "TA"<count_values>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
DESIRED_SIZE = 128
print(df.SaleType.value_counts())
 df[df.SaleType.isnull() ]<count_values>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
print(df.groupby(["Neighborhood", "SaleCondition"] ).SaleType.value_counts())
 df.loc[(df.SaleType.isnull())&(df.Neighborhood == "Sawyer")&(df.SaleCondition == "Normal"), "SaleType"] = "WD"<count_values>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
print(df.Electrical.value_counts())
 df[df.Electrical.isnull() ]<count_values>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
df[(df.YearBuilt > 2005)].Electrical.value_counts()<correct_missing_values>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
df.Electrical.fillna("SBrkr", inplace=True )<count_values>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
print(df.Exterior1st.value_counts())
 df[df.Exterior1st.isnull() ]<count_values>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
print(df[(df.RoofMatl == "Tar&Grv")].Exterior1st.value_counts())
 df.Exterior1st.fillna("Plywood", inplace=True )<count_values>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
print(df[(df.RoofMatl == "Tar&Grv")].Exterior2nd.value_counts())
 df.Exterior2nd.fillna("Plywood", inplace=True )<count_values>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
print(df.Functional.value_counts())
 df[df.Functional.isnull() ]<count_values>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
df[(df.Neighborhood == "IDOTRR")&(df.OverallQual < 5)&(df.YearRemodAdd < 1960)&(df.ExterQual == "Fa")].Functional.value_counts()<correct_missing_values>
from sklearn.linear_model import Ridge, ElasticNet
df.Functional.fillna("Mod", inplace = True )<categorify>
def metric(trueFVC, predFVC, predSTD):
clipSTD = np.clip(predSTD, 70 , 9e9)
 deltaFVC = np.clip(np.abs(trueFVC-predFVC), 0 , 1000)
 return np.mean(-1*(np.sqrt(2)*deltaFVC/clipSTD)- np.log(np.sqrt(2)*clipSTD))
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby(["Neighborhood", "LotShape", "LotConfig"])["LotFrontage"].transform("median"))
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby(["Neighborhood", "LotShape"])["LotFrontage"].transform("median"))
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby("Neighborhood")["LotFrontage"].transform("median"))<correct_missing_values>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def kloss(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 1]
fvc_pred = y_pred[:, 0]
sigma_clip = sigma + C1
delta = tf.abs(y_true[:, 0] - fvc_pred)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def kmae(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 spread = tf.abs(( y_true[:, 0] - y_pred[:, 0])/(y_pred[:, 0] + 1.))
 return K.mean(spread)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * kloss(y_true, y_pred)+(1 - _lambda)*kmae(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 preds = L.Dense(2, activation="relu", name="preds" )(x)
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.99),
optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0.01, amsgrad=False),
metrics=[kloss])
 return model
df.Utilities.fillna(df.Utilities.mode() [0], inplace=True)
 missing(df )<data_type_conversions>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
df["MSSubClass"] = df["MSSubClass"].astype("str" )<feature_engineering>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
df.loc[(df.GarageYrBlt == 2207), "GarageYrBlt"] = 2007
df.loc[(df.Exterior2nd == "CmentBd"), "Exterior2nd"] = "CemntBd"
df.loc[(df.Exterior2nd == "Wd Shng"), "Exterior2nd"] = "WdShing"
df.loc[(df.Exterior2nd == "Brk Cmn"), "Exterior2nd"] = "BrkComm"<create_dataframe>
pe0 = np.zeros(( ze.shape[0], 2))
pred0 = np.zeros(( z.shape[0], 2))
pe1 = np.zeros(( ze.shape[0], 2))
pred1 = np.zeros(( z.shape[0], 2))
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 print("===================== NEURAL NET =============================")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=200, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=500))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=500))
print("predict val...")
 pred0[val_idx] = net.predict(z[val_idx], batch_size=500, verbose=0)
 print("predict test...")
 pe0 += net.predict(ze, batch_size=500, verbose=0)/ NFOLD
print("===================== RIDGE REG =============================")
 clf = Ridge(alpha=0.05)
 clf.fit(z[tr_idx], y[tr_idx])
 pred1[val_idx, 0] = clf.predict(z[val_idx])
 pred_std = np.mean(np.abs(y[val_idx] - pred1[val_idx, 0])) * np.sqrt(2)
 pred1[val_idx, 1] = pred_std
print("val", metric(y[val_idx], pred1[val_idx, 0], pred1[val_idx, 1]))
pe1[:, 0] += clf.predict(ze)/ NFOLD
pe1[:, 1] += pred_std / NFOLD
pred0[:, 1] = pred0[:, 1] + 70 .
dff = df.copy()<count_values>
w = 0.5
pred =(1-w)* pred0 + w * pred1
pe =(1-w)* pe0 + w * pe1
pe[:, 1] = pe1[:, 1]
pred[:, 1] = pred1[:, 1]
num_cols, cat_cols, ord_cols = col_types(dff)
 for col in dff.columns:
print("For column: ", col + "
")
 print(dff[col].value_counts() , "
" )<feature_engineering>
print("oof neural net", metric(y, pred0[:, 0], pred0[:, 1]))
print("oof ridge", metric(y, pred1[:, 0], pred1[:, 1]))
print("oof ensemble", metric(y, pred[:, 0], pred[:, 1]))
dff["Older1945"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["30", "70"] else 0)
 dff["Newer1946"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["20", "60", "120", "160"] else 0)
 dff["AllStyles"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["20", "90", "190"] else 0)
 dff["AllAges"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["40", "45", "50", "75", "90", "150", "190"] else 0)
 dff["Pud"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["120", "150", "160", "180"] else 0)
 dff["Split"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["80", "85""180"] else 0)
 dff["MSSubClass"] = dff["MSSubClass"].apply(lambda x: "180" if x == "150" else x )<feature_engineering>
sigma_opt = mean_absolute_error(y, pred[:, 0])
 sigma_mean = np.mean(pred[:, 1])
 print(sigma_opt, sigma_mean )
dff["MSZoning"] = dff["MSZoning"].apply(lambda x: "R" if x.startswith("R")else x )<feature_engineering>
sub['FVC1'] = pe[:, 0]
sub['Confidence1'] = pe[:, 1]
dff["North"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Blmngtn", "BrDale", "ClearCr", "Gilbert", "Names", "NoRidge",
"NPkVill", "NWAmes", "NoRidge", "NridgHt", "Sawyer", "Somerst",
"StoneBr", "Veenker", "NridgHt"] else 0)
 dff["South"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Blueste", "Edwards", "Mitchel", "MeadowV",
"SWISU", "IDOTRR", "Timber"] else 0)
 dff["Downtown"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["BrkSide", "Crawfor", "OldTown", "CollgCr"] else 0)
 dff["East"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["IDOTRR", "Mitchel"] else 0)
 dff["West"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Edwards", "NWAmes", "SWISU", "Sawyer", "SawyerW"] else 0 )<feature_engineering>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
dff.loc[(dff["Condition1"] == "Feedr")|(dff["Condition2"] == "Feedr"), "StreetDegree"] = 1
dff.loc[(dff["Condition1"] == "Artery")|(dff["Condition2"] == "Artery"), "StreetDegree"] = 2
dff["StreetDegree"].fillna(0, inplace = True)
 dff.loc[(dff["Condition1"].isin(["RRNn", "RRNe"])) |(dff["Condition2"].isin(["RRNn", "RRNe"])) , "RailroadDegree"] = 1
dff.loc[(dff["Condition1"].isin(["RRAn", "RRAe"])) |(dff["Condition2"].isin(["RRAn", "RRAe"])) , "RailroadDegree"] = 2
dff["RailroadDegree"].fillna(0, inplace = True)
 dff.loc[(dff["Condition1"] == "PosN")|(dff["Condition2"] == "PosN"), "OffsiteFeature"] = 1
dff.loc[(dff["Condition1"] == "PosA")|(dff["Condition2"] == "PosA"), "OffsiteFeature"] = 2
dff["OffsiteFeature"].fillna(0, inplace = True)
 dff["Norm1"] = dff["Condition1"].apply(lambda x: 1 if x == "Norm" else 0)
 dff["Norm2"] = dff["Condition2"].apply(lambda x: 1 if x == "Norm" else 0)
 dff["Norm"] = dff["Norm1"] + dff["Norm2"]
dff.drop(["Norm1", "Norm2"], axis = 1, inplace = True )<define_variables>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
lotshape = {"IR3": 1, "IR2": 2, "IR1": 3, "Reg": 4}
landcontour = {"Low":1, "HLS": 2, "Bnk":3, "Lvl": 4}
utilities = {"ELO": 1, "NoSeWa": 2, "NoSewr": 3, "AllPub": 4}
landslope = {"Sev": 1, "Mod": 2, "Gtl": 3}
general = {"None": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5}
bsmtexposure = {"None": 0, "No": 0, "Mn": 1, "Av": 2, "Gd": 3}
bsmtfintype = {"None": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6}
electrical = {"Mix": 1, "FuseP": 2, "FuseF": 3, "FuseA": 4, "SBrkr": 5}
functional = {"Typ": 1, "Min1": 2, "Min2": 3, "Mod": 4, "Maj1": 5, "Maj2": 6, "Sev": 7, "Sal": 8}
garagefinish = {"None": 0, "Unf": 1, "RFn": 2, "Fin": 3}
fence = {"None": 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4}
dff.replace({"LotShape": lotshape, "LandContour": landcontour, "Utilities": utilities, "LandSlope": landslope,
"BsmtExposure": bsmtexposure, "BsmtFinType1": bsmtfintype, "BsmtFinType2":bsmtfintype, "Electrical": electrical,
"Functional": functional, "GarageFinish": garagefinish, "Fence": fence},
inplace = True)
 for col in ["ExterQual", "ExterCond", "BsmtQual", "BsmtCond", "HeatingQC", "KitchenQual",
"FireplaceQu", "GarageQual", "GarageCond", "PoolQC"]:
dff[col] = dff[col].replace(general )<feature_engineering>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
dff["BldgType"] = dff["BldgType"].apply(lambda x: "2Fam" if x in ["2fmCon", "Duplex"] else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "WD" if x.endswith("WD")else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "Contract" if x.startswith("Con")else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "Oth" if x == "COD" else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Abnormal_Adjland" if x in ["Abnorml", "AdjLand"] else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Alloca_Family" if x in ["Alloca", "Family"] else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Other" if x in ["Abnormal_Adjland", "Alloca_Family"] else x)
 dff["GarageType"] = dff["GarageType"].apply(lambda x: "Carport_None" if x in ["CarPort", "None"] else x)
 dff["GarageType"] = dff["GarageType"].apply(lambda x: "Basement_2Types" if x in ["Basment", "2Types"] else x)
 dff["LotConfig"] = dff["LotConfig"].apply(lambda x: "CulDSac_FR3" if x in ["CulDSac", "FR3"] else x)
 dff["RoofStyle"] = dff["RoofStyle"].apply(lambda x: "Other" if x not in ["Gable"] else x)
 dff["RoofMatl"] = dff["RoofMatl"].apply(lambda x: "Other" if x != "CompShg" else x)
 dff["MasVnrType"] = dff["MasVnrType"].apply(lambda x: "None_BrkCmn" if x in ["None", "BrkCmn"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "BrkTil_Stone" if x in ["BrkTil", "Stone"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "BrkTil_Stone_Slab" if x in ["BrkTil_Stone", "Slab"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "PConc_Wood" if x in ["PConc", "Wood"] else x)
 dff["Heating"] = dff["Heating"].apply(lambda x: "Other" if x != "GasA" else x )<feature_engineering>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )