Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<init_hyperparams>
copyfile(src = ".. /usr/lib/modellib/modellib.py", dst = ".. /working/ModelLib.py" )
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1500,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
import numpy as np
import pandas as pd
import torch
from torch.utils.data import TensorDataset, Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
from sklearn.model_selection import KFold,StratifiedKFold
from tqdm.auto import tqdm
from ModelLib import Create_model,stratified_group_k_fold
import random
import os
from copy import deepcopy
import math
from glob import glob
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )<save_model>
random.seed(831)
 os.environ['PYTHONHASHSEED'] = str(721)
 np.random.seed(1111)
 torch.manual_seed(1117)
 torch.cuda.manual_seed(1001)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = 'cuda'
m_lgb.save_model("model.lgb" )<count_unique_values>
train_x = np.load('.. /input/covid19fe/train_aug_x.npy')
 test_x = np.load('.. /input/covid19fe/test_aug_x.npy')
 train_bpps = np.load('.. /input/covid19fe/train_bpps.npy')
 test_bpps = np.load('.. /input/covid19fe/test_bpps.npy')
 train_viennarna_bpps = np.load('.. /input/covid19extrafeatures/train_viennarna_bpps.npy')
 test_viennarna_bpps = np.load('.. /input/covid19extrafeatures/test_viennarna_bpps.npy')
 train_mat = np.load('.. /input/covid19extrafeatures/train_mat.npy')
 test_mat = np.load('.. /input/covid19extrafeatures/test_mat.npy')
 train_aug_mat = np.load('.. /input/covid19fe/train_aug_mat.npy')
 test_aug_mat = np.load('.. /input/covid19fe/test_aug_mat.npy')
 label = np.load('.. /input/covid19fe/label.npy')
 label_error = np.load('.. /input/covid19fe/label_error.npy')
 signal_to_noise = np.load('.. /input/covid19fe/signal_to_noise.npy' )
sub.id.nunique() , sub["id"].str.contains("evaluation$" ).sum()<categorify>
train_bpps = np.concatenate([np.expand_dims(train_bpps,axis=1),np.expand_dims(train_viennarna_bpps,axis=1),np.expand_dims(train_mat,axis=1),np.expand_dims(train_aug_mat,axis=1)],axis=1)
 test_bpps = np.concatenate([np.expand_dims(test_bpps,axis=1),np.expand_dims(test_viennarna_bpps,axis=1),np.expand_dims(test_mat,axis=1),np.expand_dims(test_aug_mat,axis=1)],axis=1 )
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 20
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [30, 10, 3, 3, 5, 5]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline<load_from_csv>
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json',lines=True ).drop('index',axis=1)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json',lines=True ).drop('index',axis=1)
 train_length = train.seq_length.values
test_length = test.seq_length.values
train_scored = train.seq_scored.values
test_scored = test.seq_scored.values
SN_filter_mask =(train.SN_filter==1 ).values
SN_filter = np.where(SN_filter_mask)[0]
signal_filter = np.where(signal_to_noise > 1)[0]
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<concatenate>
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape<categorify>
class Covid19Dataset(Dataset):
def __init__(self,X,bpps,mat,seq_length,scored_length,label=None,label_error=None,signal_to_noise=None,SN_filter_mask=None):
self.X = X.astype(np.int)
 self.bpps = bpps.astype(np.float32)
 if label is not None:
self.label = label.astype(np.float32)
 self.signal_to_noise = signal_to_noise.astype(np.float32)
 self.label_error=label_error.astype(np.float32)
 self.SN_filter_mask = SN_filter_mask
else:
self.label = None
self.mask = np.zeros([len(X),130],dtype=bool)
 for i in range(len(seq_length)) :
if seq_length[i] < 130:
self.mask[i,seq_length[i]:] = True
self.scored_mask = np.ones([len(X),130],dtype=bool)
 for i in range(len(scored_length)) :
if scored_length[i] < 130:
self.scored_mask[i,scored_length[i]:] = False
self.seq_length = seq_length
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
N = self.seq_length[idx]
X = self.X[idx,:N]
bpps = self.bpps[idx,:,:N,:N]
mask = self.mask[idx,:N]
scored_mask = self.scored_mask[idx,:N]
if self.label is not None:
label = self.label[idx,:N]
label_error= self.label_error[idx,:N]
signal_to_noise = self.signal_to_noise[idx]
SN_filter_mask = self.SN_filter_mask[idx]
return X,bpps,mask,scored_mask,label,label_error,signal_to_noise,SN_filter_mask
else:
return X,bpps,mask,scored_mask
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )<compute_test_metric>
nepochs = 300
n_fold = 5
kf = StratifiedKFold(n_fold,shuffle=True,random_state=831)
 dataset = Covid19Dataset(train_x,train_bpps,train_mat,train_length,train_scored,label,label_error,signal_to_noise,SN_filter_mask)
 cv_score = []
loss_weights = torch.Tensor([1.2,1.2,1.2,0.7,0.7] ).reshape(1,5 ).to(device)
 oof = np.zeros([len(train_x),68,3])
 for fold,(trn_group, test_group)in tqdm(enumerate(kf.split(train_x,SN_filter_mask)) ,total=n_fold):
test_group = np.intersect1d(test_group,SN_filter)
 traindataset = Covid19Dataset(train_x[trn_group],
train_bpps[trn_group],
train_mat[trn_group],
train_length[trn_group],
train_scored[trn_group],
label[trn_group],
label_error[trn_group],
signal_to_noise[trn_group],
SN_filter_mask[trn_group])
 valdataset = Covid19Dataset(train_x[test_group],
train_bpps[test_group],
train_mat[test_group],
train_length[test_group],
train_scored[test_group],
label[test_group],
label_error[test_group],
signal_to_noise[test_group],
SN_filter_mask[test_group])
 args_loader = {'batch_size': 128, 'shuffle': True, 'num_workers': 0, 'pin_memory': True, 'drop_last': True}
train_loader = DataLoader(traindataset, **args_loader)
 args_loader = {'batch_size': 128, 'shuffle': False, 'num_workers': 0, 'pin_memory': True, 'drop_last': False}
val_loader = DataLoader(valdataset, **args_loader)
 dataloaders = {'train' : train_loader, 'val' : val_loader}
model,optimizer,scheduler = Create_model(device)
 best_model = {'reactivity':None,'deg_Mg_pH10':None,"deg_Mg_50C":None}
best_loss = {'reactivity': np.inf,'deg_Mg_pH10': np.inf,'deg_Mg_50C': np.inf}
for epoch in tqdm(range(nepochs)) :
epoch_loss = {'train': 0.0, 'val': 0.0, 'val_clean': 0.0, 'val_aug': 0.0,
'reactivity': 0.0,'deg_Mg_pH10': 0.0,'deg_Mg_50C': 0.0
}
MA_loss = []
test_pred = []
test_pred_aug = []
test_y = []
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
for x_b,bpps_b,mask_b,scored_mask_b,label_b,label_error_b,signal_to_noise_b,SN_filter_mask_b in dataloaders[phase]:
x_b = x_b.long().to(device)
 bpps_b = bpps_b.to(device)
 mask_b = mask_b.to(device)
 label_b = label_b.to(device)
 label_error_b = label_error_b.to(device)
 signal_to_noise_b = signal_to_noise_b.to(device ).unsqueeze(1 ).unsqueeze(1)
 signal_to_noise_b = torch.clamp(signal_to_noise_b/4.5,0,10)
 label_error_b = torch.log(1+1.0/label_error_b[:,:68])/ 2.2496114573105803
SN_filter_mask_b = SN_filter_mask_b.to(device)
 if phase=='train':
aug_mask = torch.randint(low=0,high=2,size=[len(bpps_b)],dtype=bool ).to(device)
 x_b[aug_mask,:,-4] = x_b[aug_mask,:,-2]
x_b[aug_mask,:,-3] = x_b[aug_mask,:,-1]
x_b = x_b[:,:,:-2]
bpps_b[aug_mask,-2] = bpps_b[aug_mask,-1]
bpps_b = bpps_b[:,:-1]
else:
x_b_aug = x_b.clone()
x_b_aug[:,:,-4] = x_b_aug[:,:,-2]
x_b_aug[:,:,-3] = x_b_aug[:,:,-1]
x_b_aug = x_b_aug[:,:,:-2]
bpps_b_aug = bpps_b.clone()
bpps_b_aug[:,-2] = bpps_b_aug[:,-1]
bpps_b_aug = bpps_b_aug[:,:-1]
x_b = x_b[:,:,:-2]
bpps_b = bpps_b[:,:-1]
optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
preds = model(x_b,bpps_b)
 if phase=='val':
preds_aug = model(x_b_aug,bpps_b_aug)
 preds2 = 0.5*(preds + preds_aug)
 test_pred.append(preds[:,:68,:3].detach().cpu().numpy())
 test_pred_aug.append(preds2[:,:68,:3].detach().cpu().numpy())
 test_y.append(label_b[:,:68,:3].detach().cpu().numpy())
 loss =(preds[:,:68] - label_b[:,:68])**2
loss = torch.sqrt(( loss * signal_to_noise_b ).reshape(-1,5 ).mean(0)).mean()
if phase=='train':
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if phase == 'train':
running_loss += loss.item() / len(dataloaders[phase])
 else:
running_loss += loss.item() / len(dataloaders[phase])
 if phase == 'train':
epoch_loss['train'] = running_loss
else:
epoch_loss['val'] = running_loss
test_pred = np.concatenate(test_pred,axis=0)
 test_pred_aug = np.concatenate(test_pred_aug,axis=0)
 test_y = np.concatenate(test_y,axis=0)
 epoch_loss['val_clean'] = np.sqrt(((test_pred-test_y)**2 ).reshape(-1,3 ).mean(0)).mean()
res =(( test_pred_aug-test_y)**2 ).reshape(-1,3)
 epoch_loss['val_aug'] = np.sqrt(res.mean(0)).mean()
epoch_loss['reactivity'] = np.sqrt(res[:,0].mean())
 epoch_loss['deg_Mg_pH10'] = np.sqrt(res[:,1].mean())
 epoch_loss['deg_Mg_50C'] = np.sqrt(res[:,2].mean())
 scheduler.step()
print("Epoch {}/{} - loss: {:5.5f} - val_loss: {:5.5f} - val_clean_loss: {:5.5f} - val_aug_loss: {:5.5f} - reactivity: {:5.5f} - deg_Mg_pH10: {:5.5f} - deg_Mg_50C: {:5.5f}".format(epoch+1, nepochs, epoch_loss['train'], epoch_loss['val'], epoch_loss['val_clean'], epoch_loss['val_aug'], epoch_loss['reactivity'], epoch_loss['deg_Mg_pH10'], epoch_loss['deg_Mg_50C']))
for i,cat in enumerate(['reactivity','deg_Mg_pH10','deg_Mg_50C']):
if epoch_loss[cat] < best_loss[cat]:
best_loss[cat] = epoch_loss[cat]
torch.save(model.state_dict() , f'fold{fold+1}_{cat}_model.pt')
 oof[test_group,:,i] = test_pred_aug[:,:,i]
cv_score.append(sum(best_loss.values())/3 )
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1/Ds
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 print(Ds.shape)
 return Ds
Ds = get_distance_matrix(As)
 Ds_pub = get_distance_matrix(As_pub)
 Ds_pri = get_distance_matrix(As_pri )<data_type_conversions>
for i in range(n_fold):
print(f"fold {i+1} score:",cv_score[i])
 print()
print("CV score:",np.mean(cv_score))
np.save('oof_{:5.5f}'.format(np.mean(cv_score)) ,oof )
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
As.shape, As_pub.shape, As_pri.shape<categorify>
dataset = Covid19Dataset(test_x,test_bpps,test_mat,test_length,test_scored)
 args_loader = {'batch_size': 1, 'shuffle': False, 'num_workers': 0, 'pin_memory': True, 'drop_last': False}
test_loader = DataLoader(dataset, **args_loader)
 test_predictions = np.zeros([len(test_x),130,5])
 for j,col in enumerate(['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']):
paths = glob(f'fold*_{col}_model.pt')
 with torch.no_grad() :
for path in tqdm(paths):
model.load_state_dict(torch.load(path))
model.eval().to(device)
 predictions = []
for x_b,bpps_b,mask_b,scored_mask_b in test_loader:
x_b = x_b.long().to(device)
 bpps_b = bpps_b.to(device)
 mask_b = mask_b.to(device)
 x_b_aug = x_b.clone()
x_b_aug[:,:,-4] = x_b_aug[:,:,-2]
x_b_aug[:,:,-3] = x_b_aug[:,:,-1]
x_b_aug = x_b_aug[:,:,:-2]
bpps_b_aug = bpps_b.clone()
bpps_b_aug[:,-2] = bpps_b_aug[:,-1]
bpps_b_aug = bpps_b_aug[:,:-1]
x_b = x_b[:,:,:-2]
bpps_b = bpps_b[:,:-1]
preds = model(x_b,bpps_b)
 preds_aug = model(x_b_aug,bpps_b_aug)
 preds = 0.5*(preds + preds_aug)
 p = torch.zeros([preds.shape[0],130,preds.shape[2]])
 p[:,:preds.shape[1]] = preds.cpu()
predictions.append(p)
 predictions = torch.cat(predictions,dim=0 ).numpy()
test_predictions[:,:,j] += predictions[:,:,j] / len(paths )
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )<compute_test_metric>
ss = pd.read_csv(".. /input/stanford-covid-vaccine/sample_submission.csv",index_col=0 )
def mcrmse(t, p, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
p = p[:, :seq_len_target]
score = np.mean(np.sqrt(np.mean(np.mean(( p - t)** 2, axis = 1), axis = 0)))
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(tf.reduce_mean(( t - y)** 2, axis = 1), axis = 0)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = 'tanh' )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = 'tanh' )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 128, rate = 0.4)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam<train_model>
for n,row in tqdm(test.iterrows() ,total=len(test)) :
test_id = row['id']
seq_len = row['seq_length']
for i in range(seq_len):
for j,col in enumerate(['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']):
ss.loc[test_id+'_'+str(i),col] = test_predictions[n,i,j]
<split><EOS> 
ss.to_csv("submission_cnn_{:5.5f}.csv".format(np.mean(cv_score)) ,index=True )