Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<save_to_csv>
warnings.filterwarnings('ignore')
submission=test[['ForecastId','ConfirmedCases','Fatalities']]
submission=shuffle(submission)
 submission.to_csv("submission.csv",index=False )<load_from_csv>
SEED = 731
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<data_type_conversions>
FOLDS = 6
epoch = 200
VERBOSE = 0
train.rename(columns={'Country_Region':'Countries'}, inplace=True)
 test.rename(columns={'Country_Region':'Countries'}, inplace=True)
 train.rename(columns={'Province_State':'States'}, inplace=True)
 test.rename(columns={'Province_State':'States'}, inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True )<merge>
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv')
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(df[cols].applymap(lambda seq: [token2int[x] for x in seq] ).values.tolist()), 
(0, 2, 1))
train_inputs = preprocess_inputs(train[train.signal_to_noise > 1])
 train_labels = np.array(train[train.signal_to_noise > 1][target_cols].values.tolist() ).transpose(( 0, 2, 1))
na = "na"
def filtrar_state(states, countries):
if states == na:
return countries
else:
return states<data_type_conversions>
REG=0
train_1 = train.copy()
train_1['States'].fillna(na, inplace=True)
 train_1['States'] = train_1.loc[:, ['States', 'Countries']].apply(lambda x : filtrar_state(x['States'], x['Countries']), axis=1)
 train_1.loc[:, 'Date'] = train_1.Date.dt.strftime("%m%d")
 train_1["Date"] = train_1["Date"].astype(int)
 train_1.head()<data_type_conversions>
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )
te = test.copy()
te['States'].fillna(na, inplace=True)
 te['States'] = te.loc[:, ['States', 'Countries']].apply(lambda x : filtrar_state(x['States'], x['Countries']), axis=1)
 te.loc[:, 'Date'] = te.Date.dt.strftime("%m%d")
 te["Date"] = te["Date"].astype(int)
 te.head()<categorify>
def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def build_model(gru=1,seq_len=107, pred_len=68, dropout=0.5,
embed_dim=160, hidden_dim=384):
inputs = tf.keras.layers.Input(shape=(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 if gru==0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif gru==1:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif gru==3:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==4:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif gru==5:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==6:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif gru==7:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead = tfa.optimizers.Lookahead(adam, sync_period=6)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6)
 model.compile(optimizer=adam, loss=MCRMSE)
 return model
le = preprocessing.LabelEncoder()
train_1.Countries = le.fit_transform(train_1.Countries)
 train_1['States'] = le.fit_transform(train_1['States'])
 train_1.head()<categorify>
es =tf.keras.callbacks.EarlyStopping(
 monitor='val_loss', min_delta=0, patience=20, verbose=1, mode='auto',
baseline=None, restore_best_weights=False)
 lr_callback = tf.keras.callbacks.ReduceLROnPlateau(verbose=0 )
te.Countries = le.fit_transform(te.Countries)
 te['States'] = le.fit_transform(te['States'])
 te.head()<filter>
kf = KFold(n_splits=FOLDS, shuffle=True,random_state=SEED)
 rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = 1, random_state = SEED)
 train_inputs_ = train_inputs
train_labels_ = train_labels
print(train_inputs_.shape,train_labels_.shape)
 for fold,(train_index, val_index)in enumerate(kf.split(train_inputs_,train_labels_)) :
print(f"fold {fold} has train:{len(train_index)}, val:{len(val_index)}")
 train_inputs=train_inputs_[train_index]
train_labels=train_labels_[train_index]
val_inputs=train_inputs_[val_index]
val_labels=train_labels_[val_index]
tf.keras.backend.clear_session()

 gru = build_model(gru=0)
 sv_gru = tf.keras.callbacks.ModelCheckpoint('model_gru.h5', save_best_only=True)
 history_gru = gru.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_gru,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_gru.history['loss'])}, min validation loss={min(history_gru.history['val_loss'])}")
 lstm = build_model(gru=7)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_lstm.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=1)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb1.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=2)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb2.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=3)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb3.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=4)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb4.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=5)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb5.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 lstm = build_model(gru=6)
 sv_lstm = tf.keras.callbacks.ModelCheckpoint('model_hyb6.h5', save_best_only=True)
 history_lstm = lstm.fit(
 train_inputs, train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epoch,
callbacks=[lr_callback,sv_lstm,es],
verbose = VERBOSE 
)
 print(f"Min training loss={min(history_lstm.history['loss'])}, min validation loss={min(history_lstm.history['val_loss'])}")
 public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 gru_short = build_model(gru=0, seq_len=107, pred_len=107)
 gru_long = build_model(gru=0, seq_len=130, pred_len=130)
 lstm_short = build_model(gru=7, seq_len=107, pred_len=107)
 lstm_long = build_model(gru=7, seq_len=130, pred_len=130)
 hyb1_short = build_model(gru=1, seq_len=107, pred_len=107)
 hyb1_long = build_model(gru=1, seq_len=130, pred_len=130)
 hyb2_short = build_model(gru=2, seq_len=107, pred_len=107)
 hyb2_long = build_model(gru=2, seq_len=130, pred_len=130)
 hyb3_short = build_model(gru=3, seq_len=107, pred_len=107)
 hyb3_long = build_model(gru=3, seq_len=130, pred_len=130)
 hyb4_short = build_model(gru=4, seq_len=107, pred_len=107)
 hyb4_long = build_model(gru=4, seq_len=130, pred_len=130)
 hyb5_short = build_model(gru=5, seq_len=107, pred_len=107)
 hyb5_long = build_model(gru=5, seq_len=130, pred_len=130)
 hyb6_short = build_model(gru=6, seq_len=107, pred_len=107)
 hyb6_long = build_model(gru=6, seq_len=130, pred_len=130)
 gru_short.load_weights('model_gru.h5')
 gru_long.load_weights('model_gru.h5')
 lstm_short.load_weights('model_lstm.h5')
 lstm_long.load_weights('model_lstm.h5')
 hyb1_short.load_weights('model_hyb1.h5')
 hyb1_long.load_weights('model_hyb1.h5')
 hyb2_short.load_weights('model_hyb2.h5')
 hyb2_long.load_weights('model_hyb2.h5')
 hyb3_short.load_weights('model_hyb3.h5')
 hyb3_long.load_weights('model_hyb3.h5')
 hyb4_short.load_weights('model_hyb4.h5')
 hyb4_long.load_weights('model_hyb4.h5')
 hyb5_short.load_weights('model_hyb5.h5')
 hyb5_long.load_weights('model_hyb5.h5')
 hyb6_short.load_weights('model_hyb6.h5')
 hyb6_long.load_weights('model_hyb6.h5')
 gru_public_preds = gru_short.predict(public_inputs)
 gru_private_preds = gru_long.predict(private_inputs)
 lstm_public_preds = lstm_short.predict(public_inputs)
 lstm_private_preds = lstm_long.predict(private_inputs)
 hyb1_public_preds = hyb1_short.predict(public_inputs)
 hyb1_private_preds = hyb1_long.predict(private_inputs)
 hyb2_public_preds = hyb2_short.predict(public_inputs)
 hyb2_private_preds = hyb2_long.predict(private_inputs)
 hyb3_public_preds = hyb3_short.predict(public_inputs)
 hyb3_private_preds = hyb3_long.predict(private_inputs)
 hyb4_public_preds = hyb4_short.predict(public_inputs)
 hyb4_private_preds = hyb4_long.predict(private_inputs)
 hyb5_public_preds = hyb5_short.predict(public_inputs)
 hyb5_private_preds = hyb5_long.predict(private_inputs)
 hyb6_public_preds = hyb6_short.predict(public_inputs)
 hyb6_private_preds = hyb6_long.predict(private_inputs)
 preds_gru = []
for df, preds in [(public_df, gru_public_preds),(private_df, gru_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_gru.append(single_df)
 preds_gru_df = pd.concat(preds_gru)
 preds_lstm = []
for df, preds in [(public_df, lstm_public_preds),(private_df, lstm_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_lstm.append(single_df)
 preds_lstm_df = pd.concat(preds_lstm)
 preds_hyb1 = []
for df, preds in [(public_df, hyb1_public_preds),(private_df, hyb1_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb1.append(single_df)
 preds_hyb1_df = pd.concat(preds_hyb1)
 preds_hyb2 = []
for df, preds in [(public_df, hyb2_public_preds),(private_df, hyb2_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb2.append(single_df)
 preds_hyb2_df = pd.concat(preds_hyb2)
 preds_hyb3 = []
for df, preds in [(public_df, hyb3_public_preds),(private_df, hyb3_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb3.append(single_df)
 preds_hyb3_df = pd.concat(preds_hyb3)
 preds_hyb4 = []
for df, preds in [(public_df, hyb4_public_preds),(private_df, hyb4_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb4.append(single_df)
 preds_hyb4_df = pd.concat(preds_hyb4)
 preds_hyb5 = []
for df, preds in [(public_df, hyb5_public_preds),(private_df, hyb5_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb5.append(single_df)
 preds_hyb5_df = pd.concat(preds_hyb5)
 preds_hyb6 = []
for df, preds in [(public_df, hyb6_public_preds),(private_df, hyb6_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb6.append(single_df)
 preds_hyb6_df = pd.concat(preds_hyb6)
 blend_preds_df = pd.DataFrame()
blend_preds_df['id_seqpos'] = preds_gru_df['id_seqpos']
blend_preds_df['reactivity'] = 0.2*preds_gru_df['reactivity'] + 0.2*preds_lstm_df['reactivity'] + 0.2*preds_hyb1_df['reactivity'] + 0.2*preds_hyb2_df['reactivity'] + 0.2*preds_hyb3_df['reactivity']+ 0.2*preds_hyb4_df['reactivity'] + 0.2*preds_hyb5_df['reactivity'] + 0.2*preds_hyb6_df['reactivity']
blend_preds_df['deg_Mg_pH10'] = 0.2*preds_gru_df['deg_Mg_pH10'] + 0.2*preds_lstm_df['deg_Mg_pH10'] + 0.2*preds_hyb1_df['deg_Mg_pH10'] + 0.2*preds_hyb2_df['deg_Mg_pH10'] + 0.2*preds_hyb3_df['deg_Mg_pH10']+ 0.2*preds_hyb4_df['deg_Mg_pH10'] + 0.2*preds_hyb5_df['deg_Mg_pH10'] + 0.2*preds_hyb6_df['deg_Mg_pH10']
blend_preds_df['deg_pH10'] = 0.2*preds_gru_df['deg_pH10'] + 0.2*preds_lstm_df['deg_pH10'] + 0.2*preds_hyb1_df['deg_pH10'] + 0.2*preds_hyb2_df['deg_pH10'] + 0.2*preds_hyb3_df['deg_pH10']+ 0.2*preds_hyb4_df['deg_pH10'] + 0.2*preds_hyb5_df['deg_pH10'] + 0.2*preds_hyb6_df['deg_pH10']
blend_preds_df['deg_Mg_50C'] = 0.2*preds_gru_df['deg_Mg_50C'] + 0.2*preds_lstm_df['deg_Mg_50C'] + 0.2*preds_hyb1_df['deg_Mg_50C'] + 0.2*preds_hyb2_df['deg_Mg_50C'] + 0.2*preds_hyb3_df['deg_Mg_50C']+ 0.2*preds_hyb4_df['deg_Mg_50C'] + 0.2*preds_hyb5_df['deg_Mg_50C'] + 0.2*preds_hyb6_df['deg_Mg_50C']
blend_preds_df['deg_50C'] = 0.2*preds_gru_df['deg_50C'] + 0.2*preds_lstm_df['deg_50C'] + 0.2*preds_hyb1_df['deg_50C'] + 0.2*preds_hyb2_df['deg_Mg_50C'] + 0.2*preds_hyb3_df['deg_Mg_50C']+ 0.2*preds_hyb4_df['deg_50C'] + 0.2*preds_hyb5_df['deg_Mg_50C'] + 0.2*preds_hyb6_df['deg_Mg_50C']
blend_preds_df['reactivity'] /=1.6
blend_preds_df['deg_Mg_pH10'] /=1.6
blend_preds_df['deg_pH10'] /=1.6
blend_preds_df['deg_Mg_50C']/=1.6
blend_preds_df['deg_50C'] /=1.6
sample_sub = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv')
 submission = sample_sub[['id_seqpos']].merge(blend_preds_df, on=['id_seqpos'])
 submission.head()
submission.to_csv(f"submission_{fold}.csv", index=False)
<categorify><EOS> 
files = []
for i in range(FOLDS):
csv = pd.read_csv(f'./submission_{i}.csv')
 files.append(csv)
 total = pd.concat(files, axis=1)
 total.head()
col = list(submission.columns)[1:]
print(col)
 submission1 = submission
for c in col:
submission1[c] = total[c].mean(axis=1)
 submission1.to_csv('submission.csv', index=False)
 submission1.head()