Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<define_variables>
debug = False
kfold = KFold(5, shuffle = True, random_state = 42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As)) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )<predict_on_test>
warnings.filterwarnings('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SEED = 2020
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 seed_everything(SEED )
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
class RMSELoss(nn.Module):
def __init__(self, eps=1e-6):
super().__init__()
self.mse = nn.MSELoss()
self.eps = eps
def forward(self, yhat, y):
loss = torch.sqrt(self.mse(yhat, y)+ self.eps)
 return loss
class MCRMSELoss(nn.Module):
def __init__(self, num_scored=3):
super().__init__()
self.rmse = RMSELoss()
self.num_scored = num_scored
def forward(self, yhat, y):
score = 0
for i in range(self.num_scored):
score += self.rmse(yhat[:, :, i], y[:, :, i])/ self.num_scored
return score
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<define_variables>
def load_json(path):
return pd.read_json(path, lines=True)
 df = load_json('/kaggle/input/stanford-covid-vaccine/train.json')
 df_test = load_json('/kaggle/input/stanford-covid-vaccine/test.json')
 sample_sub = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv')
 if debug:
df = df[:200]
df_test = df_test[:200]
df = df[df.SN_filter == 1]
EPOCHS = 60
BATCH_SIZE = 64
SEED = 34
TTA = True
FOLD_N = 5<define_variables>
print(set(df["sequence"].sum()))
print(set(df["structure"].sum()))
print(set(df["predicted_loop_type"].sum()))
sequence_and_structure = [i + j for i in "GACU" for j in "()."]
sequence_and_predicted_loop_type = [i + j for i in "GACU" for j in "XEMBHSI"]
structure_and_predicted_loop_type = [i + j for i in "()." for j in "XEMBHSI"]
def seed_everything(seed = SEED):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything()<categorify>
def merge_seq_seq(seq):
half = len(seq)//2
new_seq = []
for i in range(len(seq)//2):
new_seq.append(seq[i] + seq[i+half])
 return new_seq
def pandas_list_to_array(df):
return np.transpose(
 np.array(df.values.tolist()), 
(0, 2, 1)
)
 def preprocess_inputs(df, token2int, cols=['sequence', 'structure', 'predicted_loop_type']):
base_feature = np.transpose(df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values.tolist() , 
(0, 2, 1))
bpps_sum_feature = np.array(df['bpps_sum'].tolist())[:, :, np.newaxis]
bpps_max_feature = np.array(df['bpps_max'].tolist())[:, :, np.newaxis]
bpps_nb_feature = np.array(df['bpps_nb'].tolist())[:, :, np.newaxis]
return np.concatenate([base_feature, bpps_sum_feature, bpps_max_feature, bpps_nb_feature], axis=2)
 def augment_data(augment_df, df):
target_df = df.copy()
new_df = augment_df[augment_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id', 'sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id', 'cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df<compute_test_metric>
create_feture = True
preprocess_cols=["sequence", "structure", "predicted_loop_type"]
if create_feture:
df["sequence_and_structure"] =(df["sequence"] + df["structure"] ).apply(merge_seq_seq)
 df_test["sequence_and_structure"] =(df_test["sequence"] + df_test["structure"] ).apply(merge_seq_seq)
 preprocess_cols=["sequence", "structure", "predicted_loop_type", "sequence_and_structure"]
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
df['bpps_sum'] = read_bpps_sum(df)
 df_test['bpps_sum'] = read_bpps_sum(df_test)
 df['bpps_max'] = read_bpps_max(df)
 df_test['bpps_max'] = read_bpps_max(df_test)
 df['bpps_nb'] = read_bpps_nb(df)
 df_test['bpps_nb'] = read_bpps_nb(df_test)
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return tf.keras.backend.sqrt(mse)
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
def mcrmse(y_actual, y_pred, num_scored=len(target_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score<import_modules>
target_cols = ["reactivity", "deg_Mg_pH10", "deg_Mg_50C"]
tokens = [i for i in "().ACGUBEHIMSX"] + sequence_and_structure + sequence_and_predicted_loop_type + structure_and_predicted_loop_type
token2int = {x:i for i, x in enumerate(tokens)}
def preprocess_inputs(df, cols):
base_fea = np.transpose(
 np.array(
 df[cols].applymap(lambda seq: [token2int[x] for x in seq] ).values.tolist() 
),(0, 2, 1)
)
 if create_feture:
bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 else:
return base_fea
train_inputs = torch.tensor(preprocess_inputs(df, preprocess_cols)).to(device)
 print("input shape: ", train_inputs.shape)
 train_labels = torch.tensor(
 np.array(df[target_cols].values.tolist() ).transpose(0, 2, 1)
 ).float().to(device )
class GRU_Layer(Layer):
def __init__(self, hidden_dim, dropout):
super(GRU_Layer, self ).__init__()
self.custom_layers = Bidirectional(GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def call(self, inputs):
return self.custom_layers(inputs)
 class LSTM_Layer(Layer):
def __init__(self, hidden_dim, dropout):
super(LSTM_Layer, self ).__init__()
self.custom_layers = Bidirectional(LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def call(self, inputs):
return self.custom_layers(inputs )<choose_model_class>
models = ["LSTM", "LSTM_short", "GRU", "blend1", "blend2", "blend3", "blend4"]
models = [
{"model": "LSTM", "dropout": 0.4, "embed_dim": 100, "hidden_dim": 128, "hidden_layers": 3},
{"model": "LSTM", "dropout": 0.4, "embed_dim": 100, "hidden_dim": 256, "hidden_layers": 3}
]
class MIX_LSTM_GRU(tf.keras.Model):
def __init__(self, seq_len=107, pred_len=68, dropout=0.5, embed_dim=100,
hidden_dim=256, model_type=0, token2int=None):
super(MIX_LSTM_GRU, self ).__init__()
self.token2int = token2int
self.embed = L.Embedding(input_dim=len(self.token2int), output_dim=embed_dim)
 self.dense = L.Dense(5, activation='linear')
 self.pred_len = pred_len
self.hidden_dim = hidden_dim
self.dropout = dropout
self.type = model_type
def build(self, inputs):
self.gru_layers = []
self.lstm_layers = []
for i in range(2):
self.gru_layers.append(GRU_Layer(self.hidden_dim, self.dropout))
self.lstm_layers.append(LSTM_Layer(self.hidden_dim, self.dropout))
def call(self, inputs):
categorical_features = inputs[:, :, :3]
numerical_features = inputs[:, :, 3:]
embed = self.embed(categorical_features)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.concatenate([reshaped, numerical_features], axis=2)
 if self.type == 0:
hidden = self.gru_layers[0](reshaped)
 hidden = self.gru_layers[1](hidden)
 elif self.type == 1:
hidden = self.lstm_layers[0](reshaped)
 hidden = self.gru_layers[0](hidden)
 elif self.type == 2:
hidden = self.gru_layers[1](reshaped)
 hidden = self.lstm_layers[1](hidden)
 elif self.type == 3:
hidden = self.lstm_layers[0](reshaped)
 hidden = self.lstm_layers[1](hidden)
 truncated = hidden[:, :self.pred_len]
out = self.dense(truncated)
 return out<load_pretrained>
class Wave_Block(nn.Module):
def __init__(self, in_channels, out_channels, dilation_rates, kernel_size):
super(Wave_Block, self ).__init__()
self.num_rates = dilation_rates
self.convs = nn.ModuleList()
self.filter_convs = nn.ModuleList()
self.gate_convs = nn.ModuleList()
self.convs.append(nn.Conv1d(in_channels, out_channels, kernel_size=1))
dilation_rates = [2 ** i for i in range(dilation_rates)]
for dilation_rate in dilation_rates:
self.filter_convs.append(
 nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int(( dilation_rate*(kernel_size-1)) /2), dilation=dilation_rate))
self.gate_convs.append(
 nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int(( dilation_rate*(kernel_size-1)) /2), dilation=dilation_rate))
self.convs.append(nn.Conv1d(out_channels, out_channels, kernel_size=1))
def forward(self, x):
x = self.convs[0](x)
 res = x
for i in range(self.num_rates):
x = torch.tanh(self.filter_convs[i](x)) * torch.sigmoid(self.gate_convs[i](x))
x = self.convs[i + 1](x)
 res = res + torch.nn.functional.pad(x,(0, 1, 0, 0))
return res
class MyModel(nn.Module):
def __init__(
 self, seq_len=107, pred_len=68, dropout=0.4, embed_dim=100, hidden_dim=64, hidden_layers=3, main_layer="LSTM" 
):
super(MyModel, self ).__init__()
self.pred_len = pred_len
self.main_layer = main_layer
if main_layer == "GRU":
hidden_dim //= 3
self.embeding = nn.Embedding(num_embeddings=len(token2int), embedding_dim=embed_dim)
 self.lstm_layer = nn.LSTM(
 input_size=embed_dim * len(preprocess_cols)+ 3,
hidden_size=hidden_dim,
num_layers=hidden_layers,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.gru_layer = nn.GRU(
 input_size=embed_dim * len(preprocess_cols)+ 3,
hidden_size=hidden_dim,
num_layers=hidden_layers,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.hidden_lstm_layer = nn.LSTM(
 input_size=hidden_dim*2,
hidden_size=hidden_dim,
num_layers=1,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.hidden_gru_layer = nn.GRU(
 input_size=hidden_dim*2,
hidden_size=hidden_dim,
num_layers=1,
dropout=dropout,
bidirectional=True,
batch_first=True 
)
 self.wave_block = Wave_Block(hidden_dim*2, hidden_dim*2, 12, hidden_dim)
 self.linear1 = nn.Linear(hidden_dim * 2, hidden_dim)
 self.linear2 = nn.Linear(hidden_dim, hidden_dim)
 self.linear3 = nn.Linear(hidden_dim, len(target_cols))
def forward(self, seqs):
if create_feture:
categorical_feats = seqs[:, :, :len(preprocess_cols)].long()
numerical_feats = seqs[:, :, len(preprocess_cols):].float()
embed = self.embeding(categorical_feats)
 reshaped = torch.reshape(embed,(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = torch.cat([reshaped, numerical_feats], axis=2)
 else:
embed = self.embeding(seqs)
 reshaped = torch.reshape(embed,(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = torch.cat([reshaped, numerical_feats], axis=2)
 if self.main_layer == "LSTM":
output, hidden = self.lstm_layer(reshaped)
 elif self.main_layer == "GRU":
output, hidden = self.gru_layer(reshaped)
 elif self.main_layer == "blend1":
output, hidden = self.lstm_layer(reshaped)
 output, hidden = self.hidden_lstm_layer(output)
 elif self.main_layer == "blend2":
output, hidden = self.gru_layer(reshaped)
 output, hidden = self.hidden_gru_layer(output)
 elif self.main_layer == "blend3":
output, hidden = self.gru_layer(reshaped)
 output, hidden = self.hidden_gru_layer(output)
 elif self.main_layer == "blend4":
output, hidden = self.lstm_layer(reshaped)
 output, hidden = self.hidden_lstm_layer(output)
 elif self.main_layer == "WaveNet":
output, hidden = self.lstm_layer(reshaped)
 output = output.permute(0, 2, 1)
 output = self.wave_block(output)
 output = output.permute(0, 2, 1)
 truncated = output[:, : self.pred_len, :]
truncated = self.linear1(truncated)
 truncated = self.linear2(truncated)
 truncated = self.linear2(truncated)
 out = self.linear3(truncated)
 return out
criterion = MCRMSELoss(len(target_cols))
def compute_loss(batch_X, batch_Y, model, optimizer=None, is_train=True, scheduler=None):
model.train(is_train)
 pred_Y = model(batch_X)
 loss = criterion(pred_Y, batch_Y)
 if is_train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
if scheduler:
scheduler.step()
return loss.item() , pred_Y
def read_bpps_sum(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bppm = np.load(os.path.join(root_path, obj_id + '.npy')).sum(axis=1)
 bpps_arr.append(bppm)
 return bpps_arr
def read_bpps_max(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bppm = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_arr.append(np.max(bppm, axis=1))
return bpps_arr
def read_bpps_nb(df, root_path, bpps_mean, bpps_std):
bpps_arr = []
for obj_id in df.id.to_list() :
bpps = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_mean)/ bpps_std
bpps_arr.append(bpps_nb)
 return bpps_arr
def calc_bpps_mean_std(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bpps = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_arr.append(bpps_nb)
 return np.mean(bpps_arr), np.std(bpps_arr)
 def add_bpp_feature(train, test, bpps_dir):
mean, std = calc_bpps_mean_std(train, bpps_dir)
 train['bpps_sum'] = read_bpps_sum(train, bpps_dir)
 test['bpps_sum'] = read_bpps_sum(test, bpps_dir)
 train['bpps_max'] = read_bpps_max(train, bpps_dir)
 test['bpps_max'] = read_bpps_max(test, bpps_dir)
 train['bpps_nb'] = read_bpps_nb(train, bpps_dir, mean, std)
 test['bpps_nb'] = read_bpps_nb(test, bpps_dir, mean, std)
 return train, test<train_on_grid>
FOLDS = 4
EPOCHS = 100
if debug:
EPOCHS = 2
BATCH_SIZE = 64
VERBOSE = 2
LR = 0.016
def group_by_kmean_reactivity(train, token2int):
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train, token2int)[:, :, 0])
 train['cluster_id'] = kmeans_model.labels_
return train<split>
public_df = df_test.query("seq_length == 107" ).copy()
private_df = df_test.query("seq_length == 130" ).copy()
public_inputs = torch.tensor(preprocess_inputs(public_df, preprocess_cols)).to(device)
 private_inputs = torch.tensor(preprocess_inputs(private_df, preprocess_cols)).to(device)
 public_loader = DataLoader(TensorDataset(public_inputs), shuffle=False, batch_size=BATCH_SIZE)
 private_loader = DataLoader(TensorDataset(private_inputs), shuffle=False, batch_size=BATCH_SIZE)
def train_mix_lstm_gru(train, public_df, private_df, target_cols, token2int, model_type=0, FOLD_N=FOLD_N):
Ver='MIX_LSTM_GRU_' + str(model_type)
 gkf = GroupKFold(n_splits=FOLD_N)
 public_inputs = preprocess_inputs(public_df, token2int)
 private_inputs = preprocess_inputs(private_df, token2int)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn, token2int)
 y_trn = np.array(trn[target_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val, token2int)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val, token2int)
 y_val = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
model = MIX_LSTM_GRU(model_type=model_type, token2int=token2int)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 model_short = MIX_LSTM_GRU(seq_len=107, pred_len=107, model_type=model_type, token2int=token2int)
 model_long = MIX_LSTM_GRU(seq_len=130, pred_len=130, model_type=model_type, token2int=token2int)
 history = model.fit(
 x_trn, y_trn,
validation_data =(x_val, y_val),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5')
 ] 
)
 fig = px.line(
 history.history, y=['loss', 'val_loss'],
labels={'index': 'epoch', 'value': 'Mean Squared Error'},
title='Training History')
 fig.show()
model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_short.compile(optimizer=tf.optimizers.Adam() , loss=mcrmse)
 model_short.train_on_batch(tf.zeros(( 1,107,6)) , tf.zeros(( 1,107,5)))
 model_short.load_weights(f'model{Ver}_cv{cv}.h5')
 model_long.compile(optimizer=tf.optimizers.Adam() , loss=mcrmse)
 model_long.train_on_batch(tf.zeros(( 1,130,6)) , tf.zeros(( 1,130,5)))
 model_long.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
public_preds = model_short.predict(public_inputs)/FOLD_N
private_preds = model_long.predict(private_inputs)/FOLD_N
else:
public_preds += model_short.predict(public_inputs)/FOLD_N
private_preds += model_long.predict(private_inputs)/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds<load_from_csv>
if debug:
kmeans_model = KMeans(n_clusters=50, random_state=110 ).fit(preprocess_inputs(df, preprocess_cols)[:,:,0])
 else:
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(df, preprocess_cols)[:,:,0])
 kmeans_labels = kmeans_model.labels_
data_dir = '/kaggle/input/stanford-covid-vaccine/'
bpps_dir = '/kaggle/input/stanford-covid-vaccine/bpps/'
train = pd.read_json(data_dir + 'train.json', lines=True)
 test = pd.read_json(data_dir + 'test.json', lines=True)
 sample_df = pd.read_csv(data_dir + 'sample_submission.csv')
 aug_df = pd.read_csv('/kaggle/input/openvaccine-augment/aug_data.csv')
 token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
train, test = add_bpp_feature(train, test, bpps_dir)
 train = group_by_kmean_reactivity(train, token2int)
 if TTA:
train = augment_data(aug_df, train)
 test = augment_data(aug_df, test)
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
train_inputs = preprocess_inputs(train, token2int)
 train_labels = pandas_list_to_array(train[target_cols])
 public_df = test.query("seq_length == 107")
 private_df = test.query("seq_length == 130" )<train_on_grid>
model_histories = {str(model_id): [] for model_id in models}
model_oof_preds = {str(model_id): np.zeros(( df.shape[0], 68, len(target_cols)))for model_id in models}
model_private_preds = {str(model_id): np.zeros(( private_df.shape[0], 130, len(target_cols)))for model_id in models}
model_public_preds = {str(model_id): np.zeros(( public_df.shape[0], 107, len(target_cols)))for model_id in models}
criterion = MCRMSELoss()
gkf = GroupKFold(FOLDS)
 for model_dict in models:
model_id = str(model_dict)
 model_name = model_dict["model"]
dropout = model_dict["dropout"]
embed_dim = model_dict["embed_dim"]
hidden_layers = model_dict["hidden_layers"]
model_oof_pred = []
oof_idxes = []
for k,(train_index, val_index)in enumerate(gkf.split(train_inputs, df["reactivity"], kmeans_labels)) :
oof_idxes.append(val_index)
 train_dataset = TensorDataset(train_inputs[train_index], train_labels[train_index])
 val_dataset = TensorDataset(train_inputs[val_index], train_labels[val_index])
 train_loader = DataLoader(train_dataset, shuffle=True, batch_size=BATCH_SIZE)
 val_loader = DataLoader(val_dataset, shuffle=True, batch_size=BATCH_SIZE)
 model = MyModel(dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 optimizer = optim.Adam(model.parameters() , lr=LR)
 scheduler = None
scheduler = CosineAnnealingLR(optimizer, T_max=20, eta_min=0.008)
 train_losses = []
val_losses = []
for epoch in tqdm(range(EPOCHS)) :
train_losses_batch = []
val_losses_batch = []
for(batch_X, batch_Y)in train_loader:
train_loss, _ = compute_loss(batch_X, batch_Y, model, optimizer=optimizer, is_train=True, scheduler=scheduler)
 train_losses_batch.append(train_loss)
 for(batch_X, batch_Y)in val_loader:
model.eval()
val_loss, val_pred = compute_loss(batch_X, batch_Y, model, optimizer=optimizer, is_train=False)
 val_losses_batch.append(val_loss)
 train_losses.append(np.mean(train_losses_batch))
val_losses.append(np.mean(val_losses_batch))
model_state = model.state_dict()
torch.save(model_state, f"{model_id}_fold{k}.pth")
 del model
model_histories[model_id].append({"train_loss": train_losses, "val_loss": val_losses})
 model_short = MyModel(seq_len=107, pred_len=107, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_short.load_state_dict(model_state)
 model_short.eval()
model_public_pred = np.ndarray(( 0, 107, len(target_cols)))
 for batch in public_loader:
batch_X = batch[0]
pred = model_short(batch_X ).detach().cpu().numpy()
model_public_pred = np.concatenate([model_public_pred, pred], axis=0)
 model_public_preds[model_id] += model_public_pred / FOLDS
model_long = MyModel(seq_len=130, pred_len=130, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_long.load_state_dict(model_state)
 model_long.eval()
model_private_pred = np.ndarray(( 0, 130, len(target_cols)))
 for batch in private_loader:
batch_X = batch[0]
pred = model_long(batch_X ).detach().cpu().numpy()
model_private_pred = np.concatenate([model_private_pred, pred], axis=0)
 model_private_preds[model_id] += model_private_pred / FOLDS
oof_loader = DataLoader(val_dataset, shuffle=False, batch_size=BATCH_SIZE)
 model_oof = MyModel(seq_len=107, pred_len=68, dropout=dropout, embed_dim=embed_dim, hidden_layers=hidden_layers, main_layer=model_name ).to(device)
 model_oof.load_state_dict(model_state)
 model_oof.eval()
oof_pred = np.ndarray(( 0, 68, len(target_cols)))
 for batch in oof_loader:
batch_X = batch[0]
pred = model_oof(batch_X ).detach().cpu().numpy()
oof_pred = np.concatenate([oof_pred, pred], axis=0)
 model_oof_pred.append(oof_pred)
 del model_short, model_long
gc.collect()
oof_idxes = np.concatenate(oof_idxes)
 order = np.argsort(oof_idxes)
 model_oof_preds[model_id] = np.concatenate(model_oof_pred)[order]
val_df, val_preds, test_df, test_preds = [], [], [], []
debug = False
if debug:
nmodel = 1
else:
nmodel = 4
for i in range(nmodel):
holdouts, holdout_preds, public_df, public_preds, private_df, private_preds = train_mix_lstm_gru(train,
public_df, private_df,
target_cols,
token2int,model_type=i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(public_df)
 test_df.append(private_df)
 test_preds.append(public_preds)
 test_preds.append(private_preds )<prepare_output>
public_df = df_test.query("seq_length == 107" ).copy()
private_df = df_test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df, preprocess_cols)
 private_inputs = preprocess_inputs(private_df, preprocess_cols )
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()<save_to_csv>
preds_model = {str(model_id): [] for model_id in models}
submissions = {}
for model_dict in models:
model_id = str(model_dict)
 for df, preds in [(public_df, model_public_preds[model_id]),(private_df, model_private_preds[model_id])]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model[model_id].append(single_df)
 preds_df = pd.concat(preds_model[model_id])
 print(preds_df.head())
 submission = sample_sub[['id_seqpos']].merge(preds_df, on=['id_seqpos'])
 submission['deg_pH10'] = 0
submission['deg_50C'] = 0
print(submission.head())
 submission.to_csv(f'submission_{model_id}.csv', index=False)
 submissions[model_id] = submission
print('Submission saved')
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )<import_modules>
if True:
id_seqpos = submissions[str(models[0])]["id_seqpos"]
ensemble_submission = submissions[str(models[0])].drop("id_seqpos",axis=1)*best_alpha + submissions[str(models[1])].drop("id_seqpos",axis=1)*(1-best_alpha)
 ensemble_submission["id_seqpos"] = id_seqpos
ensemble_submission.to_csv(f'submission_lstm_gru_ensemble.csv', index=False )
<import_modules><EOS> 
if not debug:
!curl -X POST -H 'Content-type: application/json' --data '{"text":"commit done! "}' <your_webhook_url>