Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<choose_model_class>
import pandas as pd
import numpy as np
import json
import os
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras.utils.vis_utils import plot_model
import tensorflow.keras.layers as L
import keras.backend as K
import tensorflow as tf
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
!pip install spektral -q
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
from spektral.layers import GraphConv
def model_gru_atten_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<predict_on_test>
train_json_path = "/kaggle/input/stanford-covid-vaccine/train.json"
test_json_path = "/kaggle/input/stanford-covid-vaccine/test.json"
sample_sub_path = "/kaggle/input/stanford-covid-vaccine/sample_submission.csv"
output_path = "./"
bpps_path = "/kaggle/input/stanford-covid-vaccine/bpps"
train_df = pd.read_json(train_json_path, lines=True)
 test_df = pd.read_json(test_json_path, lines=True)
 public_df = test_df.query("seq_length == 107" ).copy()
private_df = test_df.query("seq_length == 130" ).copy()
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<load_pretrained>
train_y = np.array(train_df[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
train_y.shape
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
vocab = []
for w,k in word_index.items() :
vocab.append(w)
 if k >= max_features:
break
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<define_variables>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
sequence_token2int = {x:i for i, x in enumerate('AGUC')}
structure_token2int = {
'.': 0,
'(': 1,
')': 2,
}
loop_token2int = {x:i for i, x in enumerate('SMIBHEX')}
token2int_map = {
"sequence": sequence_token2int,
"structure": structure_token2int,
"predicted_loop_type": loop_token2int
}
sequence_columns = ["sequence", "structure", "predicted_loop_type"]
def to_seq(df):
return np.transpose(
 np.array(
 df[sequence_columns] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 train = to_seq(train_df)
 public = to_seq(public_df)
 private = to_seq(private_df)
 train.shape, public.shape, private.shape
outputs = []<predict_on_test>
def to_one_hot(df):
temp = np.transpose(
 np.array([
df[col] 
.apply(lambda seq: [token2int_map[col][x] for x in seq])
.values 
.tolist()
for col in sequence_columns
]), 
(1, 2, 0)
)
 ohe_1 = tf.keras.utils.to_categorical(temp[:,:,0], 4)
 ohe_2 = tf.keras.utils.to_categorical(temp[:,:,1], 3)
 ohe_3 = tf.keras.utils.to_categorical(temp[:,:,2], 7)
 return np.concatenate([ohe_1, ohe_2, ohe_3], axis=2)
 train_ohe = to_one_hot(train_df)
 public_ohe = to_one_hot(public_df)
 private_ohe = to_one_hot(private_df)
 train_ohe.shape, public_ohe.shape, private_ohe.shape
pred_val_y, pred_test_y = train_pred(model_gru_atten_3(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '3 GRU w/ atten'] )<predict_on_test>
def get_adjacency_matrix(inps):
As = []
for row in range(0, inps.shape[0]):
A = np.zeros(( inps.shape[1], inps.shape[1]))
stack = []
opened_so_far = []
for seqpos in range(0, inps.shape[1]):
if inps[row, seqpos, 1] == 0:
stack.append(seqpos)
 opened_so_far.append(seqpos)
 elif inps[row, seqpos, 1] == 1:
openpos = stack.pop()
A[openpos, seqpos] = 1
A[seqpos, openpos] = 1
As.append(A)
 return np.array(As)
 train_adj = get_adjacency_matrix(train)
 public_adj = get_adjacency_matrix(public)
 private_adj = get_adjacency_matrix(private)
 train_adj.shape, public_adj.shape, private_adj.shape
pred_val_y, pred_test_y = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, 'gru atten srk'] )<predict_on_test>
def get_bpps(mRNA_ids):
bpps = []
for mRNA_id in tqdm(mRNA_ids):
bpps.append(
 np.load(f"{bpps_path}/{mRNA_id}.npy"), 
)
 return np.array(bpps)
 train_bpps = get_bpps(train_df.id.values)
 public_bpps = get_bpps(public_df.id.values)
 private_bpps = get_bpps(private_df.id.values)
 train_bpps.shape, public_bpps.shape, private_bpps.shape
pred_val_y, pred_test_y = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, '2d CNN GloVe'] )<predict_on_test>
train_bpps_stats = [train_bpps.mean(axis=2), train_bpps.max(axis=2)]
public_bpps_stats = [public_bpps.mean(axis=2), public_bpps.max(axis=2)]
private_bpps_stats = [private_bpps.mean(axis=2), private_bpps.max(axis=2)]
pred_val_y, pred_test_y = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, 'LSTM DU'] )<predict_on_test>
train_bpps_stats = np.concatenate([stats[:,:,None] for stats in train_bpps_stats], axis=2)
 public_bpps_stats = np.concatenate([stats[:,:,None] for stats in public_bpps_stats], axis=2)
 private_bpps_stats = np.concatenate([stats[:,:,None] for stats in private_bpps_stats], axis=2)
 train_bpps_stats.shape, public_bpps_stats.shape, private_bpps_stats.shape
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<predict_on_test>
scored_seq_length = 68
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred):
score = 0
for i in range(y_actual.shape[2]):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ y_actual.shape[2]
return score
def build_model(input_seq_len=107, output_seq_len=scored_seq_length):
def _bi_gru_block(x, hidden_dim, dropout):
gru = L.Bidirectional(
 L.GRU(hidden_dim,
dropout=dropout,
return_sequences=True, 
),
)(x)
 return gru
def _conv_block(x, adj_m, bpp_m, conv_filters, graph_channels):
conv = L.Conv1D(
 conv_filters, 5,
padding='same',
activation='tanh',
)(x)
 gcn_1 = GraphConv(
 graph_channels,
activation='tanh',
)([conv, adj_m])
 gcn_2 = GraphConv(
 graph_channels,
activation='tanh',
)([conv, bpp_m])
 conv = L.Concatenate()([conv, gcn_1, gcn_2])
 conv = L.Activation("relu" )(conv)
 conv = L.SpatialDropout1D(0.1 )(conv)
 return conv
one_hot_encoding_inputs = L.Input(shape=(input_seq_len, 14), name="onehot")
 adj_matrix_inputs = L.Input(( input_seq_len, input_seq_len), name="adjmatrix")
 base_pair_proba_inputs = L.Input(( input_seq_len, input_seq_len), name="pairproba")
 base_pair_proba_stats_inputs = L.Input(shape=(input_seq_len, 2), name="pairprobastats")
 merged_inputs = L.Concatenate()([one_hot_encoding_inputs, base_pair_proba_stats_inputs])
 hidden = _conv_block(merged_inputs, adj_matrix_inputs, base_pair_proba_inputs, 512, 80)
 hidden = _bi_gru_block(hidden, 256, 0.5)
 hidden = _conv_block(hidden, adj_matrix_inputs, base_pair_proba_inputs, 512, 80)
 hidden = _bi_gru_block(hidden, 256, 0.5)
 out = hidden[:, :output_seq_len]
out = L.Dense(5, activation='linear' )(out)
 model = tf.keras.Model(
 inputs=[
one_hot_encoding_inputs,
adj_matrix_inputs,
base_pair_proba_inputs,
base_pair_proba_stats_inputs,
],
outputs=out, 
)
 return model
model = build_model()
plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention GloVe'] )<predict_on_test>
split_results = train_test_split(
 train_ohe,
train_adj,
train_bpps,
train_bpps_stats,
train_y,
train_df.signal_to_noise,
train_df.SN_filter,
test_size=0.1,
random_state=42, 
)
 [a.shape for a in split_results]
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention Para'] )<load_from_csv>
trn_ohe, val_ohe, trn_adj, val_adj, trn_bpps, val_bpps, trn_bpps_stats, val_bpps_stats, trn_y, val_y, trn_snr, val_snr, trn_snf, val_snf = split_results
coefs = [0.20076554,0.07993707,0.11611663,0.14885248,0.15734404,0.17454667,0.14288361]
pred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.34 ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<set_options>
model = build_model()
model.compile(tf.keras.optimizers.Adam() , loss=mcrmse )
tqdm.pandas()<init_hyperparams>
trn_inputs = [trn_ohe, trn_adj, trn_bpps, trn_bpps_stats]
val_inputs = [val_ohe, val_adj, val_bpps, val_bpps_stats]
emb_size = 300
max_features = 200000
maxlen = 100<set_options>
val_mask = np.where(( val_snf==1))
val_inputs = [val_input[val_mask] for val_input in val_inputs]
val_y = val_y[val_mask]
def clean_memory(*args):
for arg in args:
del arg
gc.collect()
time.sleep(10 )<load_from_csv>
sample_weight = np.log(trn_snr+1.11)/2
path = '/kaggle/input/quora-insincere-questions-classification'
train_df = pd.read_csv(path + "/train.csv")
 test_df = pd.read_csv(path + "/test.csv")
 train_df["question_text"].fillna("_na_", inplace=True)
 test_df["question_text"].fillna("_na_", inplace=True )<feature_engineering>
history = model.fit(
 trn_inputs, trn_y,
validation_data =(val_inputs, val_y),
batch_size=64,
epochs=300,
sample_weight=sample_weight,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(verbose=1, monitor='val_loss'),
tf.keras.callbacks.ModelCheckpoint(f'model.h5',save_best_only=True, verbose=0, monitor='val_loss'),
tf.keras.callbacks.EarlyStopping(
 patience=20,
monitor='val_loss',
verbose=0,
mode="auto",
baseline=None,
restore_best_weights=True, 
),
],
verbose=2 
)
 print(f"Min validation loss history={min(history.history['val_loss'])}" )
train_df["num_words"] = train_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["question_text"].progress_apply(lambda x: len(str(x)))
<feature_engineering>
model.load_weights(f'model.h5' )
test_df["num_words"] = test_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
test_df["num_unique_words"] = test_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 test_df["num_chars"] = test_df["question_text"].progress_apply(lambda x: len(str(x)) )<feature_engineering>
val_preds = model.predict(val_inputs)
 tf.reduce_mean(mcrmse(val_y, val_preds))
def build_vocab(sentences, verbose = True):
vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<feature_engineering>
model_public = build_model(107, 107)
 model_private = build_model(130, 130)
 model_public.load_weights(f'model.h5')
 model_private.load_weights(f'model.h5' )
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
def load_all() :
word2vec_format = {}
glove = [o.split(" ")[0] for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) ]
for word in tqdm(glove):
word2vec_format[word] = 1
clean_memory(glove)
 paragram = [o.split(" ")[0] for o in open(emb_path + '/paragram_300_sl999/paragram_300_sl999.txt', encoding="utf8", errors='ignore')if len(o)>100]
para = 0
for word in tqdm(paragram):
word2vec_format[word] = 1
clean_memory(paragram)
 return word2vec_format
emb_all = load_all()<categorify>
public_inputs = [public_ohe, public_adj, public_bpps, public_bpps_stats,]
private_inputs = [private_ohe, private_adj, private_bpps, private_bpps_stats,]
def check_coverage(vocab, embeddings_index):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x<split>
test_preds = [model_public.predict(public_inputs), model_private.predict(private_inputs)]
test_dfs = [public_df, private_df]
test_preds[0].shape, test_preds[1].shape
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]<define_variables>
preds_ls = []
for df, preds in zip(test_dfs, test_preds):
for i, uid in tqdm(enumerate(df.id)) :
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
test_df.shape, preds_df.shape
<feature_engineering><EOS> 
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )