Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<merge>
warnings.simplefilter('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 quantiles = [0.2, 0.5, 0.8]
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new<categorify>
def make_X(dt, dense_cols, cat_feats):
X = {"dense": dt[dense_cols].to_numpy() }
for i, v in enumerate(cat_feats):
X[v] = dt[[v]].to_numpy()
return X
class Loader:
def __init__(self, X, y, shuffle=True, batch_size=64, cat_cols=[]):
self.X_cont = X["dense"]
self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)
 self.y = y
self.shuffle = shuffle
self.batch_size = batch_size
self.n_conts = self.X_cont.shape[1]
self.len = self.X_cont.shape[0]
n_batches, remainder = divmod(self.len, self.batch_size)
 if remainder > 0:
n_batches += 1
self.n_batches = n_batches
self.remainder = remainder
self.idxes = np.array([i for i in range(self.len)])
 def __iter__(self):
self.i = 0
if self.shuffle:
ridxes = self.idxes
np.random.shuffle(ridxes)
 self.X_cat = self.X_cat[[ridxes]]
self.X_cont = self.X_cont[[ridxes]]
if self.y is not None:
self.y = self.y[[ridxes]]
return self
def __next__(self):
if self.i >= self.len:
raise StopIteration
if self.y is not None:
y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))
else:
y = None
xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])
 xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])
 batch =(xcont, xcat, y)
 self.i += self.batch_size
return batch
def __len__(self):
return self.n_batches
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 fillin_columns(test_new,"key", 'rate_ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed_rate)
 fillin_columns(test_new,"key", 'rate_Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality_rate )<categorify>
class model_nn(nn.Module):
def __init__(self, hidden_dim, output_dim, emb_dims, n_cont):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 n_embs = sum([y for x, y in emb_dims])
 self.n_embs = n_embs
self.n_cont = n_cont
inp_dim = n_embs + n_cont
self.inp_dim = inp_dim
self.fs0 = nn.Linear(inp_dim, hidden_dim)
 self.relufs0 = nn.ELU()
self.fs1 = nn.Linear(hidden_dim, inp_dim)
 self.fs2 = nn.Sigmoid()
self.fc0 = nn.Linear(inp_dim, hidden_dim)
 self.relu0 = nn.ELU()
self.fc1 = nn.Linear(hidden_dim, hidden_dim)
 self.relu1 = nn.ELU()
self.fc2 = nn.Linear(hidden_dim, output_dim)
 self.fc3 = nn.Linear(hidden_dim, output_dim)
 self.fc4 = nn.Linear(hidden_dim, output_dim)
 def encode_and_combine_data(self, cat_data):
xcat = [el(cat_data[:, k])for k, el in enumerate(self.emb_layers)]
xcat = torch.cat(xcat, 1)
 return xcat
def forward(self, cont_data, cat_data):
cont_data = cont_data.to(device)
 cat_data = cat_data.to(device)
 cat_data = self.encode_and_combine_data(cat_data)
 x = torch.cat([cont_data, cat_data], dim=1)
 w = self.fs0(x)
 w = self.relufs0(w)
 w = self.fs1(w)
 w = self.fs2(w)
 wx = w*x + x
hz = self.fc0(wx)
 hz = self.relu0(hz)
 hz = self.fc1(hz)
 hz = self.relu1(hz)
 out1 = self.fc2(hz)
 out2 = self.fc3(hz)
 out3 = self.fc4(hz)
 return torch.cat([out1, out2, out3], dim=1 )
<train_model>
def quantile_loss(preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss
def metric(outputs, target):
confidence = np.abs(outputs[:, 2] - outputs[:, 0])
 clip = np.where(confidence > 70, confidence, 70)
 delta = np.abs(outputs[:, 1] - target)
 delta = np.where(delta > 1000, 1000, delta)
 metrics =(delta*np.sqrt(2)/clip)+ np.log(clip*np.sqrt(2))
return np.mean(metrics )
def bagged_set_trainc(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 30,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<define_variables>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
Model_Root = 'models'
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age_week'] = data['Age'].values + data['base_week'].values/53
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['min_percent_norm'] =(data['min_percent'] - data['min_percent'].min())/(data['min_percent'].max() - data['min_percent'].min())
 data['age_week_norm'] =(data['age_week'] - data['age_week'].min())/(data['age_week'].max() - data['age_week'].min())
 cat_feat = ['Sex', 'SmokingStatus']
uniques = []
for i, v in enumerate(cat_feat):
data[v] = OrdinalEncoder(dtype="int" ).fit_transform(data[[v]])
 uniques.append(len(data[v].unique()))
tr = data.loc[data.WHERE == 'train']
chunk = data.loc[data.WHERE == 'val']
sub = data.loc[data.WHERE == 'test']
FE += ['age', 'week', 'BASE', 'min_percent_norm', 'age_week_norm']
print(list(tr))
print(list(sub))
print(FE)
 print(cat_feat)
 print(uniques)
 dims = [1, 2]
emb_dims = [(x, y)for x, y in zip(uniques, dims)]
n_cont = len(FE )
<categorify><EOS> 
hidden_dim = 256
out_dim = 1
kfold = 6
groups = np.asarray(tr['Patient'].values)
 skf = GroupKFold(n_splits=kfold)
 avg_preds = np.zeros(( len(sub), len(quantiles)))
 models = []
for i,(train_index, test_index)in enumerate(skf.split(tr, tr['FVC'].values, groups=groups)) :
print('[Fold %d/%d]' %(i + 1, kfold))
model_path = f"{Model_Root}/nn_model_%s.pth" % i
if os.path.isfile(model_path):
final_model = torch.load(model_path)
 X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
print(X_cont.shape, X_cat.shape)
 out = final_model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
else:
X_train, X_valid = tr.iloc[train_index], tr.iloc[test_index]
y_train, y_valid = tr.iloc[train_index]['FVC'].values, tr.iloc[test_index]['FVC'].values
X_train = make_X(X_train.reset_index() , FE, cat_feat)
 X_valid = make_X(X_valid.reset_index() , FE, cat_feat)
 train_loader = Loader(X_train, y_train, cat_cols=cat_feat, batch_size=16, shuffle=True)
 val_loader = Loader(X_valid, y_valid, cat_cols=cat_feat, batch_size=64, shuffle=True)
 model = model_nn(hidden_dim, out_dim, emb_dims, n_cont ).to(device)
 final_model = model_training(model, train_loader, val_loader, epochs=1000,
batch_size=64, lr=0.01, patience=20,
model_path='nn_model_%s.pth' % i)
 models.append(final_model)
 for model in models:
X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
with torch.no_grad() :
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
out = model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
print(avg_preds)
 avg_preds = avg_preds/kfold
print(avg_preds)
 sub['Patient_Week'] = sub['Patient_Week'].values
sub['FVC'] = avg_preds[:, 1]
sub['Confidence'] = avg_preds[:, 2]-avg_preds[:, 0]
sub[['Patient_Week', 'FVC', 'Confidence']].to_csv("submission.csv", index=False )