Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<count_missing_values>
warnings.filterwarnings("ignore" )
test_df.isna().sum()<drop_column>
seed = 42
train_df['Country_Region'] = train_df['Country_Region'] + ' ' + train_df['Province_State']
test_df['Country_Region'] = test_df['Country_Region'] + ' ' + test_df['Province_State']
del train_df['Province_State']
del test_df['Province_State']<feature_engineering>
DEVICE = "TPU"
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
train_df.Date = train_df.Date.apply(split_date)
 test_df.Date = test_df.Date.apply(split_date )<feature_engineering>
dropout_model = 0.36
hidden_dim_first = 128
hidden_dim_second = 256
hidden_dim_third = 128
year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
train = pd.read_json('.. /input/bpps-data-included/out_train(1 ).json')
 test = pd.read_json('.. /input/bpps-data-included/out_test(1 ).json')
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv" )
train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']<feature_engineering>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols=['sequence','predicted_loop_type','structure']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
_, position_fea = np.mgrid[0:bpps_nb_fea.shape[0]:1, 0:bpps_nb_fea.shape[1]:1]/(bpps_nb_fea.shape[1]-1)
 return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea, position_fea[:,:,np.newaxis]], 2)
 kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
aug_df = pd.read_csv('.. /input/augmented-data/aug_data1.csv')
 display(aug_df.head() )
test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']<drop_column>
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )
del train_df['Year']
del test_df['Year']<data_type_conversions>
train_inputs_all = preprocess_inputs(train)
 train_labels_all = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))
train_df['ConfirmedCases'] = train_df['ConfirmedCases'].apply(int)
 train_df['Fatalities'] = train_df['Fatalities'].apply(int )<drop_column>
def scaled_dot_product_attention(q, k, v, mask):

 matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(k)[-1], tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
 self.wk = tf.keras.layers.Dense(d_model)
 self.wv = tf.keras.layers.Dense(d_model)
 self.dense = tf.keras.layers.Dense(d_model)
 def split_heads(self, x, batch_size):

 x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(
 q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention, 
(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights
def get_config(self):
config = super().get_config().copy()
config.update({
'depth': self.depth,
'wq': self.wq,
'qk': self.wk,
'wv': self.wv,
'dense': self.dense,
})
 return config
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model)
 ])
 class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(EncoderLayer, self ).__init__()
self.d_model = d_model
self.num_heads = num_heads
self.dff = dff
self.rate = rate
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, training):
attn_output, _ = self.mha(x, x, x, None)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
def get_config(self):
config = super().get_config().copy()
config.update({
'num_heads': self.num_heads,
'rate': self.rate,
'd_model': self.d_model,
'num_heads': self.num_heads,
'dropout1': self.dropout1,
'dropout2': self.dropout2,
'layernorm1': self.layernorm1,
'layernorm2': self.layernorm2,
'mha': self.mha,
'ffn': self.ffn,
})
 return config
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )
cases = train_df.ConfirmedCases
fatalities = train_df.Fatalities
del train_df['ConfirmedCases']
del train_df['Fatalities']<categorify>
def build_model(model_type=1, seq_len=107, pred_len=68, embed_dim=200,
dropout=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third):
inputs = tf.keras.layers.Input(shape=(seq_len, 7))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:6]
positional_fea = tf.expand_dims(inputs[:, :, 6], axis=2)
 embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 reshaped = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 conv = L.Conv1D(255, 5, padding='same', activation=tf.keras.activations.swish )(reshaped)
 reshaped = L.concatenate([conv, positional_fea], axis=2)
 hidden = EncoderLayer(256, 128, 512 )(reshaped)
 hidden = EncoderLayer(256, 128, 512 )(hidden)
 if model_type == 0:
hidden = gru_layer(256, 0.3 )(hidden)
 hidden = gru_layer(256, 0.3 )(hidden)
 elif model_type == 1:
hidden = lstm_layer(256, 0.3 )(hidden)
 hidden = gru_layer(256, 0.3 )(hidden)
 elif model_type == 2:
hidden = gru_layer(256, 0.3 )(hidden)
 hidden = lstm_layer(256, 0.3 )(hidden)
 elif model_type == 3:
hidden = lstm_layer(256, 0.3 )(hidden)
 hidden = lstm_layer(256, 0.3 )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(len(target_cols), activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer=adam, loss=MCRMSE, metrics=[tf.keras.metrics.RootMeanSquaredError() ])
 return model
lb = LabelEncoder()
train_df['Country_Region'] = lb.fit_transform(train_df['Country_Region'])
 test_df['Country_Region'] = lb.transform(test_df['Country_Region'] )<normalization>
tf.keras.backend.clear_session()
lr_callback = tf.keras.callbacks.ReduceLROnPlateau()
es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',restore_best_weights=True,min_delta=0.001, patience=10 )
scaler = MinMaxScaler()
x_train = scaler.fit_transform(train_df.values)
 x_test = scaler.transform(test_df.values )<import_modules>
mse_s = []
rmse_s = []
def train_and_predict(n_folds=5, model_name="model", model_type=0, epochs=100, debug=True,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed):
print("Model:", model_name)
 ensemble_preds = pd.DataFrame(index=sample_sub.index, columns=target_cols ).fillna(0)
 kf = KFold(n_folds, shuffle=True, random_state=seed)
 skf = StratifiedKFold(n_folds, shuffle=True, random_state=seed)
 gkf = GroupKFold(n_splits=n_folds)
 val_losses = []
historys = []
for i,(train_index, val_index)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
print("Fold:", str(i+1))
with strategy.scope() :
model_train = build_model(model_type=model_type,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_short = build_model(model_type=model_type, seq_len=107, pred_len=107,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_long = build_model(model_type=model_type, seq_len=130, pred_len=130,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 train_inputs, train_labels = train_inputs_all[train_index], train_labels_all[train_index]
val = train.iloc[val_index]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
val_inputs = preprocess_inputs(val)
 val_labels = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(train.iloc[train_index].signal_to_noise+1.1)/2
checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{model_name}_Fold_{str(i+1)}.h5')
 history = model_train.fit(
 train_inputs , train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
sample_weight=w_trn/2,
epochs=epochs,
callbacks=[checkpoint,
lr_callback,
TqdmCallback() ,
tf.keras.callbacks.TerminateOnNaN() ,
es_callback],
verbose= 0 
)
 holdouts = train.iloc[val_index]
holdout_preds = model_train.predict(x_val_all)
 holdout_labels = np.array(holdouts[target_cols].values.tolist() ).transpose(( 0, 2, 1))
rmse =(( holdout_labels - holdout_preds)** 2 ).mean() **.5
mse =(( holdout_labels - holdout_preds)** 2 ).mean()
print(f"{model_name} Min training loss={min(history.history['loss'])}, min validation loss={min(history.history['val_loss'])}")
 print(f"{model_name} Holdouts mse ={mse}, Holdouts rmse ={rmse}")
 mse_s.append(mse)
 rmse_s.append(rmse)
 mse_s_t.append(mse)
 rmse_s_t.append(rmse)
 val_losses.append(min(history.history['val_loss']))
historys.append(history)
 model_short.load_weights(f'{model_name}_Fold_{str(i+1)}.h5')
 model_long.load_weights(f'{model_name}_Fold_{str(i+1)}.h5')
 public_preds = model_short.predict(public_inputs)
 private_preds = model_long.predict(private_inputs)
 preds_model = []
for df, preds in [(public_df, public_preds),(private_df, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model.append(single_df)
 preds_model_df = pd.concat(preds_model ).groupby('id_seqpos' ).mean().reset_index()
ensemble_preds[target_cols] += preds_model_df[target_cols].values / n_folds
if debug:
print("Intermediate ensemble result")
 print(ensemble_preds[target_cols].head())
 ensemble_preds["id_seqpos"] = preds_model_df["id_seqpos"].values
ensemble_preds = pd.merge(sample_sub["id_seqpos"], ensemble_preds, on="id_seqpos", how="left")
 print("Mean Validation loss:", str(np.mean(val_losses)))
 if debug:
fig, ax = plt.subplots(1, 3, figsize =(20, 10))
for i, history in enumerate(historys):
ax[0].plot(history.history['loss'])
 ax[0].plot(history.history['val_loss'])
 ax[0].set_title('model_'+str(i+1))
ax[0].set_ylabel('Loss')
 ax[0].set_xlabel('Epoch')
 ax[1].plot(history.history['root_mean_squared_error'])
 ax[1].plot(history.history['val_root_mean_squared_error'])
 ax[1].set_title('model_'+str(i+1))
ax[1].set_ylabel('RMSE')
 ax[1].set_xlabel('Epoch')
 ax[2].plot(history.history['lr'])
 ax[2].set_title('model_'+str(i+1))
ax[2].set_ylabel('LR')
 ax[2].set_xlabel('Epoch')
 plt.show()
return ensemble_preds
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 ensembles = []
for i in range(1):
model_name = "model_"+str(i+1)
 mse_s_t = []
rmse_s_t = []
ensemble = train_and_predict(n_folds=5, model_name=model_name, model_type=i, epochs=60,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed)
 ensembles.append(ensemble)
 print("RMSE Avg ", np.array(rmse_s_t ).mean())
 print("MSE Avg ", np.array(mse_s_t ).mean() )
from xgboost import XGBRegressor<train_model>
ensemble_final = ensembles[0].copy()
ensemble_final[target_cols] = 0
for ensemble in ensembles:
ensemble_final[target_cols] += ensemble[target_cols].values / len(ensembles)
 ensemble_final.head().T
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train,cases )<predict_on_test>
blend_preds_df = pd.DataFrame()
blend_preds_df['id_seqpos'] = ensemble_final['id_seqpos']
blend_preds_df['reactivity'] = ensemble_final['reactivity']
blend_preds_df['deg_Mg_pH10'] = ensemble_final['deg_Mg_pH10']
blend_preds_df['deg_pH10'] = ensemble_final['deg_Mg_pH10']
blend_preds_df['deg_Mg_50C'] = ensemble_final['deg_Mg_50C']
blend_preds_df['deg_50C'] = ensemble_final['deg_Mg_50C']
blend_preds_df.head().T
<feature_engineering><EOS> 
blend_preds_df.to_csv('submission.csv', index=False )