Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<categorify>
warnings.simplefilter('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 quantiles = [0.2, 0.5, 0.8]
y = train_df['target'] 
<prepare_x_and_y>
def make_X(dt, dense_cols, cat_feats):
X = {"dense": dt[dense_cols].to_numpy() }
for i, v in enumerate(cat_feats):
X[v] = dt[[v]].to_numpy()
return X
class Loader:
def __init__(self, X, y, shuffle=True, batch_size=64, cat_cols=[]):
self.X_cont = X["dense"]
self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)
 self.y = y
self.shuffle = shuffle
self.batch_size = batch_size
self.n_conts = self.X_cont.shape[1]
self.len = self.X_cont.shape[0]
n_batches, remainder = divmod(self.len, self.batch_size)
 if remainder > 0:
n_batches += 1
self.n_batches = n_batches
self.remainder = remainder
self.idxes = np.array([i for i in range(self.len)])
 def __iter__(self):
self.i = 0
if self.shuffle:
ridxes = self.idxes
np.random.shuffle(ridxes)
 self.X_cat = self.X_cat[[ridxes]]
self.X_cont = self.X_cont[[ridxes]]
if self.y is not None:
self.y = self.y[[ridxes]]
return self
def __next__(self):
if self.i >= self.len:
raise StopIteration
if self.y is not None:
y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))
else:
y = None
xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])
 xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])
 batch =(xcont, xcat, y)
 self.i += self.batch_size
return batch
def __len__(self):
return self.n_batches
y_val=val_df['target']<choose_model_class>
class model_nn(nn.Module):
def __init__(self, hidden_dim, output_dim, emb_dims, n_cont):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 n_embs = sum([y for x, y in emb_dims])
 self.n_embs = n_embs
self.n_cont = n_cont
inp_dim = n_embs + n_cont
self.inp_dim = inp_dim
self.fc0 = nn.Linear(inp_dim, hidden_dim)
 self.relu0 = nn.ELU()
self.fc1 = nn.Linear(hidden_dim, hidden_dim)
 self.relu1 = nn.ELU()
self.fc2 = nn.Linear(hidden_dim, output_dim)
 self.fc3 = nn.Linear(hidden_dim, output_dim)
 self.fc4 = nn.Linear(hidden_dim, output_dim)
 def encode_and_combine_data(self, cat_data):
xcat = [el(cat_data[:, k])for k, el in enumerate(self.emb_layers)]
xcat = torch.cat(xcat, 1)
 return xcat
def forward(self, cont_data, cat_data):
cont_data = cont_data.to(device)
 cat_data = cat_data.to(device)
 cat_data = self.encode_and_combine_data(cat_data)
 x = torch.cat([cont_data, cat_data], dim=1)
 hz = self.fc0(x)
 hz = self.relu0(hz)
 hz = self.fc1(hz)
 hz = self.relu1(hz)
 out1 = self.fc2(hz)
 out2 = self.fc3(hz)
 out3 = self.fc4(hz)
 return torch.cat([out1, out2, out3], dim=1 )
file_path = "model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 10 )<import_modules>
def quantile_loss(preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss
def metric(outputs, target):
confidence = np.abs(outputs[:, 2] - outputs[:, 0])
 clip = np.where(confidence > 70, confidence, 70)
 delta = np.abs(outputs[:, 1] - target)
 delta = np.where(delta > 1000, 1000, delta)
 metrics =(delta*np.sqrt(2)/clip)+ np.log(clip*np.sqrt(2))
return np.mean(metrics)
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU, CuDNNGRU, CuDNNLSTM, BatchNormalization
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D
from keras.models import Model, load_model
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras import backend as K
from keras.engine import InputSpec, Layer
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback, EarlyStopping<set_options>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
Model_Root = 'models'
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age_week'] = data['Age'].values + data['base_week'].values/53
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['min_percent_norm'] =(data['min_percent'] - data['min_percent'].min())/(data['min_percent'].max() - data['min_percent'].min())
 data['age_week_norm'] =(data['age_week'] - data['age_week'].min())/(data['age_week'].max() - data['age_week'].min())
 cat_feat = ['Sex', 'SmokingStatus']
uniques = []
for i, v in enumerate(cat_feat):
data[v] = OrdinalEncoder(dtype="int" ).fit_transform(data[[v]])
 uniques.append(len(data[v].unique()))
tr = data.loc[data.WHERE == 'train']
chunk = data.loc[data.WHERE == 'val']
sub = data.loc[data.WHERE == 'test']
FE += ['age', 'week', 'BASE', 'min_percent_norm', 'age_week_norm']
print(list(tr))
print(list(sub))
print(FE)
 print(cat_feat)
 print(uniques)
 dims = [1, 2]
emb_dims = [(x, y)for x, y in zip(uniques, dims)]
n_cont = len(FE )
<feature_engineering><EOS> 
hidden_dim = 256
out_dim = 1
kfold = 6
groups = np.asarray(tr['Patient'].values)
 skf = GroupKFold(n_splits=kfold)
 avg_preds = np.zeros(( len(sub), len(quantiles)))
 models = []
for i,(train_index, test_index)in enumerate(skf.split(tr, tr['FVC'].values, groups=groups)) :
print('[Fold %d/%d]' %(i + 1, kfold))
model_path = f"{Model_Root}/nn_model_%s.pth" % i
if os.path.isfile(model_path):
final_model = torch.load(model_path)
 X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
print(X_cont.shape, X_cat.shape)
 out = final_model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
else:
X_train, X_valid = tr.iloc[train_index], tr.iloc[test_index]
y_train, y_valid = tr.iloc[train_index]['FVC'].values, tr.iloc[test_index]['FVC'].values
X_train = make_X(X_train.reset_index() , FE, cat_feat)
 X_valid = make_X(X_valid.reset_index() , FE, cat_feat)
 train_loader = Loader(X_train, y_train, cat_cols=cat_feat, batch_size=16, shuffle=True)
 val_loader = Loader(X_valid, y_valid, cat_cols=cat_feat, batch_size=64, shuffle=True)
 model = model_nn(hidden_dim, out_dim, emb_dims, n_cont ).to(device)
 final_model = model_training(model, train_loader, val_loader, epochs=1000,
batch_size=64, lr=0.01, patience=20,
model_path='nn_model_%s.pth' % i)
 models.append(final_model)
 for model in models:
X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
with torch.no_grad() :
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
out = model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
print(avg_preds)
 avg_preds = avg_preds/kfold
print(avg_preds)
 sub['Patient_Week'] = sub['Patient_Week'].values
sub['FVC'] = avg_preds[:, 1]
sub['Confidence'] = avg_preds[:, 2]-avg_preds[:, 0]
sub[['Patient_Week', 'FVC', 'Confidence']].to_csv("submission.csv", index=False)