Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<import_modules>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.linear_model import Lasso
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeRegressor
from lightgbm import LGBMRegressor
from feature_engine import creation
from feature_engine import discretisation as disc
from feature_engine import encoding as enc
from feature_engine import imputation as imp
from feature_engine import selection as sel<load_from_csv>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<split>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
X_train, X_test, y_train, y_test = train_test_split(data.drop(
 ['Id', 'SalePrice'], axis=1),
data['SalePrice'],
test_size=0.05,
random_state=0)
 X_train.shape, X_test.shape<drop_column>
seed_everything(42 )
id_ = submission['Id']
submission.drop('Id', axis=1, inplace=True)
 submission.shape<categorify>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
def create_master_data(train, test, submission, y_train, y_test):
train = train.copy()
test = test.copy()
submission = submission.copy()
train['target'] = y_train
train['data'] = 'train'
train.reset_index(drop=True, inplace=True)
 test['target'] = y_test
test['data'] = 'test'
test.reset_index(drop=True, inplace=True)
 submission['target'] = np.nan
submission['data'] = 'submission'
submission.reset_index(drop=True, inplace=True)
 master_data = pd.concat([train, test, submission], axis=0)
 master_data.reset_index(drop=True, inplace=True)
 return master_data<prepare_x_and_y>
Dropout_model = 0.385
FVC_weight = 0.2
Confidence_weight = 0.2
y_train = np.log(y_train)
 y_test = np.log(y_test )<feature_engineering>
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )
X_train['HasPool'] = np.where(X_train['PoolArea']>0,1,0)
 X_test['HasPool'] = np.where(X_test['PoolArea']>0,1,0)
 submission['HasPool'] = np.where(submission['PoolArea']>0,1,0)
 X_train['HasAlley'] = np.where(X_train['Alley'].isnull() , 0, 1)
 X_test['HasAlley'] = np.where(X_test['Alley'].isnull() , 0, 1)
 submission['HasAlley'] = np.where(submission['Alley'].isnull() , 0, 1 )<feature_engineering>
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111
X_train['Condition_total'] = np.where(X_train['Condition2'] == X_train['Condition1'], 0, 1)
 X_test['Condition_total'] = np.where(X_test['Condition2'] == X_test['Condition1'], 0, 1)
 submission['Condition_total'] = np.where(submission['Condition2'] == submission['Condition1'], 0, 1)
 X_train['Condition_total'].value_counts()<feature_engineering>
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105
X_train['Exterior_total'] = np.where(X_train['Exterior1st'] == X_train['Exterior2nd'], 0, 1)
 X_test['Exterior_total'] = np.where(X_test['Exterior1st'] == X_test['Exterior2nd'], 0, 1)
 submission['Exterior_total'] = np.where(submission['Exterior1st'] == submission['Exterior2nd'], 0, 1)
 X_train['Exterior_total'].value_counts()<define_variables>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158
categorical = [var for var in data.columns if data[var].dtype == 'O']
categorical = categorical + ['MSSubClass']
len(categorical )<data_type_conversions>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107
X_train[categorical] = X_train[categorical].astype('O')
 X_test[categorical] = X_test[categorical].astype('O')
 submission[categorical] = submission[categorical].astype('O' )<normalization>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125
drop_features = sel.DropFeatures(features_to_drop = ['Alley', 'MiscFeature', 'PoolQC'])
 X_train = drop_features.fit_transform(X_train)
 X_test = drop_features.transform(X_test)
 submission = drop_features.transform(submission)
 X_train.shape, X_test.shape, submission.shape<train_model>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089
cat_imputer = imp.CategoricalImputer(return_object=True)
 cat_imputer.fit(X_train)
 cat_imputer.variables[0:10]<define_variables>
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100
len(cat_imputer.variables )<categorify>
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096
X_train = cat_imputer.transform(X_train)
 X_test = cat_imputer.transform(X_test)
 submission = cat_imputer.transform(submission)
 X_train.shape, X_test.shape, submission.shape<count_missing_values>
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100
[c for c in cat_imputer.variables if X_train[c].isnull().sum() >0]<define_variables>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
categorical = cat_imputer.variables
len(categorical )<train_model>
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093
constant = sel.DropConstantFeatures(tol=0.94, variables=categorical)
 constant.fit(X_train)
 constant.features_to_drop_<concatenate>
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283
data = create_master_data(X_train, X_test, submission, y_train, y_test)
 data.shape<categorify>
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087
rare_enc = enc.RareLabelEncoder(tol = 0.1,
n_categories=1,
variables=constant.features_to_drop_,
replace_with='Other', 
)
 rare_enc.fit(X_train )<categorify>
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
X_train = rare_enc.transform(X_train)
 X_test = rare_enc.transform(X_test)
 submission = rare_enc.transform(submission )<categorify>
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
qual_mappings = {'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5, 'Missing': 0, 'NA': 0}
qual_vars = ['ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond',
'HeatingQC', 'KitchenQual', 'FireplaceQu',
'GarageQual', 'GarageCond',
]
for var in qual_vars:
X_train[var] = X_train[var].map(qual_mappings)
 X_test[var] = X_test[var].map(qual_mappings)
 submission[var] = submission[var].map(qual_mappings )<categorify>
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093
exposure_mappings = {'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4, 'Missing': 0, 'NA': 0}
var = 'BsmtExposure'
X_train[var] = X_train[var].map(exposure_mappings)
 X_test[var] = X_test[var].map(exposure_mappings)
 submission[var] = submission[var].map(exposure_mappings )<categorify>
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402
finish_mappings = {'Missing': 0, 'NA': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}
finish_vars = ['BsmtFinType1', 'BsmtFinType2']
for var in finish_vars:
X_train[var] = X_train[var].map(finish_mappings)
 X_test[var] = X_test[var].map(finish_mappings)
 submission[var] = submission[var].map(finish_mappings )<categorify>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1
garage_mappings = {'Missing': 0, 'NA': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
var = 'GarageFinish'
X_train[var] = X_train[var].map(garage_mappings)
 X_test[var] = X_test[var].map(garage_mappings)
 submission[var] = submission[var].map(garage_mappings )<categorify>
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0
fence_mappings = {'Missing': 0, 'NA': 0, 'MnWw': 1, 'GdWo': 2, 'MnPrv': 3, 'GdPrv': 4}
var = 'Fence'
X_train[var] = X_train[var].map(fence_mappings)
 X_test[var] = X_test[var].map(fence_mappings)
 submission[var] = submission[var].map(fence_mappings )<define_variables>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
qual_vars = qual_vars + finish_vars + ['BsmtExposure','GarageFinish','Fence']<define_variables>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
categorical = [c for c in categorical if c not in constant.features_to_drop_]
categorical = [c for c in categorical if c not in qual_vars]
len(categorical )<groupby>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
def calculate_mean_target_per_category(df, var):
df = pd.concat([df, y_train], axis=1)
 total_houses = len(df)
 temp_df = pd.Series(df[var].value_counts() / total_houses ).reset_index()
temp_df.columns = [var, 'perc_houses']
temp_df = temp_df.merge(df.groupby([var])['SalePrice'].mean().reset_index() ,
on=var,
how='left')
 return temp_df<groupby>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
temp_df = calculate_mean_target_per_category(X_train, 'Neighborhood')
 temp_df.head()<feature_engineering>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
X_train['Neigh_Price'] = np.where(X_train['Neighborhood'].isin(cheap_neighbourhoods),
0, np.where(X_train['Neighborhood'].isin(expensive_neighbourhoods),
2, 1))
X_test['Neigh_Price'] = np.where(X_test['Neighborhood'].isin(cheap_neighbourhoods),
0, np.where(X_test['Neighborhood'].isin(expensive_neighbourhoods),
2, 1))
submission['Neigh_Price'] = np.where(submission['Neighborhood'].isin(cheap_neighbourhoods),
0, np.where(submission['Neighborhood'].isin(expensive_neighbourhoods),
2, 1))
X_train['Neigh_Price'].value_counts()<categorify>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
rare_enc = enc.RareLabelEncoder(tol = 0.05,
n_categories=4,
variables=categorical 
)
 rare_enc.fit(X_train )<categorify>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
X_train = rare_enc.transform(X_train)
 X_test = rare_enc.transform(X_test)
 submission = rare_enc.transform(submission)
 X_train.shape, X_test.shape, submission.shape<concatenate>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
data = create_master_data(X_train, X_test, submission, y_train, y_test )<train_model>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
selector = sel.SelectByTargetMeanPerformance(
 variables=categorical,
scoring='r2_score',
threshold=0.1,
cv=2,
random_state=0)
 selector.fit(X_train, y_train )<define_variables>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
len(selector.features_to_drop_ )<categorify>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
X_train = selector.transform(X_train)
 X_test = selector.transform(X_test)
 submission = selector.transform(submission)
 X_train.shape, X_test.shape, submission.shape<define_variables>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
categorical = [var for var in X_train.columns if X_train[var].dtype == 'O']
categorical = categorical + ['MSSubClass']
len(categorical)
<data_type_conversions>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
X_train[categorical] = X_train[categorical].astype('O')
 X_test[categorical] = X_test[categorical].astype('O')
 submission[categorical] = submission[categorical].astype('O' )<categorify>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
encoder = enc.OrdinalEncoder(encoding_method='ordered',
variables=categorical)
 encoder.fit(X_train, y_train )<categorify>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
encoder.encoder_dict_<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
X_train = encoder.transform(X_train)
 X_test = encoder.transform(X_test)
 submission = encoder.transform(submission)
 X_train.shape, X_test.shape, submission.shape<count_missing_values>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
[c for c in categorical if X_train[c].isnull().sum() >0]<count_missing_values>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
[c for c in categorical if X_test[c].isnull().sum() >0]<count_missing_values>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
[c for c in categorical if submission[c].isnull().sum() >0]<define_variables>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
year_vars = [var for var in data.columns if 'Yr' in var or 'Year' in var]
year_vars<compute_test_metric>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
data = create_master_data(X_train, X_test, submission, y_train, y_test )<categorify>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
create = creation.CombineWithReferenceFeature(
 variables_to_combine = ['YrSold'],
reference_variables = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt'],
operations=['sub'], 
)
 create.fit(X_train )<categorify>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
X_train = create.transform(X_train)
 X_test = create.transform(X_test)
 submission = create.transform(submission)
 X_train.shape, X_test.shape, submission.shape<categorify>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
remodelled = creation.CombineWithReferenceFeature(
 variables_to_combine = ['YearRemodAdd'],
reference_variables = ['YearBuilt'],
operations=['sub'], 
)
 remodelled.fit(X_train )<categorify>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
X_train = remodelled.transform(X_train)
 X_test = remodelled.transform(X_test)
 submission = remodelled.transform(submission)
 X_train.shape, X_test.shape, submission.shape<normalization>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
drop_features = sel.DropFeatures(features_to_drop =['YearBuilt', 'YearRemodAdd', 'GarageYrBlt','YrSold'])
 X_train = drop_features.fit_transform(X_train)
 X_test = drop_features.transform(X_test)
 submission = drop_features.transform(submission)
 X_train.shape, X_test.shape, submission.shape<count_missing_values>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
X_train[[
'YrSold_sub_YearBuilt', 'YrSold_sub_YearRemodAdd' ,'YrSold_sub_GarageYrBlt', 'YearRemodAdd_sub_YearBuilt']
].isnull().mean()<count_missing_values>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
X_test[[
'YrSold_sub_YearBuilt', 'YrSold_sub_YearRemodAdd' ,'YrSold_sub_GarageYrBlt', 'YearRemodAdd_sub_YearBuilt']
].isnull().mean()<count_missing_values>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
submission[[
'YrSold_sub_YearBuilt', 'YrSold_sub_YearRemodAdd' ,'YrSold_sub_GarageYrBlt', 'YearRemodAdd_sub_YearBuilt']
].isnull().mean()<count_unique_values>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
discrete = [
var for var in X_train.columns if X_train[var].dtype != 'O'
and var not in year_vars
and var not in qual_vars+categorical
and len(X_train[var].unique())< 20
]
len(discrete )<categorify>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
drop_features = sel.DropFeatures(features_to_drop =['PoolArea'])
 X_train = drop_features.fit_transform(X_train)
 X_test = drop_features.transform(X_test)
 submission = drop_features.transform(submission)
 X_train.shape, X_test.shape, submission.shape<define_variables>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<train_model><EOS> 
df.to_csv('submission.csv', index=False )