Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<define_variables>
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 20
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [30, 10, 3, 3, 5, 5]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline
embed_size = 300
max_features = len(word_dict)+1
maxlen = 72<prepare_x_and_y>
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )
X = pad_sequences(train_word_sequences, maxlen=maxlen)
 X_test = pad_sequences(test_word_sequences, maxlen=maxlen)
 Y = df_train['target'].values<load_pretrained>
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_embeddings(word_dict, lemma_dict, 'glove')
 embedding_matrix_para, nb_words = load_embeddings(word_dict, lemma_dict, 'para')
 embedding_matrix = np.mean(( 1.28*embedding_matrix_glove, 0.72*embedding_matrix_para), axis=0)
 del embedding_matrix_glove, embedding_matrix_para
gc.collect()
embedding_matrix.shape<choose_model_class>
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.RandomUniform(seed=10000)
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1/Ds
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 print(Ds.shape)
 return Ds
Ds = get_distance_matrix(As)
 Ds_pub = get_distance_matrix(As_pub)
 Ds_pri = get_distance_matrix(As_pri )
def LSTM_GRU(spatialdropout=0.2, rnn_units=64, weight_decay=0.07):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 x = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 rnn2 = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(rnn1)
 x = concatenate([rnn1, rnn2])
 x = GlobalMaxPooling1D()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=111100))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay),)
 return model<choose_model_class>
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
As.shape, As_pub.shape, As_pri.shape
def poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], input_length=maxlen, trainable=False, )(x_input)
 x = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNGRU(gru_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 last = Lambda(lambda t: t[:, -1], name='last' )(rnn1)
 maxpool = GlobalMaxPooling1D()(rnn1)
 attn = AttentionWeightedAverage()(rnn1)
 average = GlobalAveragePooling1D()(rnn1)
 c = concatenate([last, maxpool, attn], axis=1)
 c = Reshape(( 3, -1))(c)
 c = Lambda(lambda x: K.sum(x, axis=1))(c)
 x = BatchNormalization()(c)
 x = Dense(200, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=111000))(x)
 x = Dropout(0.2, seed=1024 )(x)
 x = BatchNormalization()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=111000))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model<choose_model_class>
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )
def BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 80, 30, 12], weight_decay=0.10):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 x = SpatialDropout1D(rate=spatialdropout, seed=10000 )(emb)
 rnn = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=123000), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 x1 = Conv1D(filters=filters[0], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=110000))(rnn)
 x2 = Conv1D(filters=filters[1], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=120000))(rnn)
 x3 = Conv1D(filters=filters[2], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=130000))(rnn)
 x4 = Conv1D(filters=filters[3], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=140000))(rnn)
 x1 = GlobalMaxPooling1D()(x1)
 x2 = GlobalMaxPooling1D()(x2)
 x3 = GlobalMaxPooling1D()(x3)
 x4 = GlobalMaxPooling1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=111000))(c)
 x = Dropout(0.2, seed=10000 )(x)
 x = BatchNormalization()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=110000))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model<compute_test_metric>
def mcrmse(t, p, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
p = p[:, :seq_len_target]
score = np.mean(np.sqrt(np.mean(np.mean(( p - t)** 2, axis = 1), axis = 0)))
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(tf.reduce_mean(( t - y)** 2, axis = 1), axis = 0)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 256, kernel = 3, rate = 0.2)
 x2 = forward(x1, 128, kernel = 6, rate = 0.1)
 x3 = forward(x2, 64, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 256, rate = 0.3)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<train_model>
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 ae_model.fit([X_node, As], [X_node[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 ae_model.fit([X_node_pub, As_pub], [X_node_pub[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 ae_model.fit([X_node_pri, As_pri], [X_node_pri[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("./base_ae" )
f1, threshold = f1_smart(np.squeeze(Y[val_list]), np.squeeze(oof[val_list]))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))<save_to_csv>
kfold = KFold(5, shuffle = True, random_state = 42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As)) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )
y_test = y_test.reshape(( -1, 1))
submission = df_test[['qid']].copy()
submission['prediction'] =(y_test>threshold ).astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]
<load_from_csv><EOS> 
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()