Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<predict_on_test>
!pip install torchcontrib
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
import os
import math
import time
import torch
import random
import shutil
import datetime
import functools
import numpy as np
import pandas as pd
import torch.nn.functional as F
from torch import nn
from pathlib import Path
from torchcontrib.optim import SWA
from sklearn.cluster import KMeans
from collections import OrderedDict
from torch.utils.data import Dataset
from sklearn.model_selection import *
from transformers import get_linear_schedule_with_warmup
from torch.nn import TransformerEncoder, TransformerEncoderLayer
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df = preds_df[['id_seqpos','reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C']]
preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<import_modules>
TARGETS = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
SCORED_TARGETS = [0, 1, 3]
NUM_TARGETS = len(TARGETS)
 SEQ_SCORED_PUBLIC = 68
SEQ_SCORED_PRIVATE = 91
SEQ_LEN_PUBLIC = 107
SEQ_LEN_PRIVATE = 130
import os
import random
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
import torch.optim.lr_scheduler as lr
from sklearn.model_selection import KFold, GroupKFold
from torch.utils.data import DataLoader, Dataset
from tqdm.notebook import tqdm
import itertools
from sklearn.cluster import KMeans<set_options>
BASE_PATH = ".. /input/stanford-covid-vaccine/"
CP_PATH = ""
PRETRAINED_PATH = ".. /input/covid-pretrained/pretrained_model.pt"
DEVICE = torch.device('cuda')
 TODAY = str(datetime.date.today() )
def seed_everything(seed):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False<define_variables>
train_df = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 test_df = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = test_df[test_df["seq_length"] == SEQ_LEN_PUBLIC].reset_index(drop=True)
 private_df = test_df[test_df["seq_length"] == SEQ_LEN_PRIVATE].reset_index(drop=True )
token2int = {x: i for i, x in enumerate("().AGUCSMIBHEX ")}<load_from_disk>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
input_cols = ['sequence', 'structure', 'predicted_loop_type']
error_cols = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_Mg_50C', 'deg_error_pH10', 'deg_error_50C']
token_dicts = {
"sequence": {x: i for i, x in enumerate("ACGU")},
"structure": {x: i for i, x in enumerate('().')},
"predicted_loop_type": {x: i for i, x in enumerate("BEHIMSX")}
}
train = pd.read_json(".. /input/stanford-covid-vaccine/train.json", lines=True)
 test = pd.read_json(".. /input/stanford-covid-vaccine/test.json", lines=True )<train_model>
def set_seed(seed=42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
SEED = 1234
set_seed(SEED )
temp = np.array(train["sequence"].apply(lambda seq: [token2int[x] for x in seq] ).values.tolist())
 kmeans = KMeans(n_clusters=200, random_state=42 ).fit(temp)
 train["cluster"] = kmeans.labels_<load_pretrained>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
 )
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train["bpps_sum"] = read_bpps_sum(train)
 train["bpps_max"] = read_bpps_max(train)
 train["bpps_nb"] = read_bpps_nb(train)
 test["bpps_sum"] = read_bpps_sum(test)
 test["bpps_max"] = read_bpps_max(test)
 test["bpps_nb"] = read_bpps_nb(test )<define_variables>
X = preprocess_inputs(train_df, cols=['sequence'])[:, :, 0]
kmeans = KMeans(n_clusters=200, random_state=110 ).fit(X)
 groups = kmeans.labels_
targets = ["reactivity", "deg_Mg_pH10", "deg_pH10", "deg_Mg_50C", "deg_50C"]<feature_engineering>
aug_df = pd.read_csv('.. /input/covid-data/aug_data.csv' )
train["seq_masked_1"] = train.apply(lambda x: "".join([y if x.structure[i] == "." else " " for i, y in enumerate(x.sequence)]), axis=1)
 train["seq_masked_2"] = train.apply(lambda x: "".join([y if x.structure[i] == "(" else " " for i, y in enumerate(x.sequence)]), axis=1)
 train["seq_masked_3"] = train.apply(lambda x: "".join([y if x.structure[i] == ")" else " " for i, y in enumerate(x.sequence)]), axis=1)
 test["seq_masked_1"] = test.apply(lambda x: "".join([y if x.structure[i] == "." else " " for i, y in enumerate(x.sequence)]), axis=1)
 test["seq_masked_2"] = test.apply(lambda x: "".join([y if x.structure[i] == "(" else " " for i, y in enumerate(x.sequence)]), axis=1)
 test["seq_masked_3"] = test.apply(lambda x: "".join([y if x.structure[i] == ")" else " " for i, y in enumerate(x.sequence)]), axis=1 )<prepare_x_and_y>
def augment_data(df, concat=True):
df = df.copy()
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left' ).sort_values('index')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
if concat:
df = df.append(new_df[df.columns] ).reset_index(drop=True)
 return df
else:
return new_df[df.columns].reset_index(drop=True )
class MyDataset(Dataset):
def __init__(self, df, mode="train"):
self.df = df.reset_index(drop=True)
 self.mode = mode
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
x1 = np.array(self.df.loc[idx, ["sequence", "structure", "predicted_loop_type", "seq_masked_1", "seq_masked_2", "seq_masked_3"]].apply(lambda seq: [token2int[x] for x in seq] ).values.tolist() ).transpose(1, 0)
 x2 = np.array(self.df.loc[idx, ["bpps_sum", "bpps_max", "bpps_nb"]].values.tolist() ).transpose(1, 0)
 if self.mode == "train":
return x1, x2, np.array(self.df.loc[idx, targets].values.tolist() ).transpose(1, 0)
 elif self.mode == "test":
return x1, x2
else:
return None<choose_model_class>
PUBLIC_IDS = public_df['id'].values
def is_public(id_seqpos):
id_ = '_'.join(id_seqpos.split('_')[:2])
 return id_ in PUBLIC_IDS
class MyModel(nn.Module):
def __init__(self, args):
super().__init__()
self.embedding = nn.Embedding(num_embeddings=args["num_embeddings"], embedding_dim=args["embedding_dim"])
 self.lstm = nn.LSTM(input_size=args["in_features"] * args["embedding_dim"] + 3, hidden_size=args["lstm_hidden"], num_layers=args["lstm_layers"], bias=True, batch_first=True, dropout=args["lstm_dropout"], bidirectional=True)
 self.fc = nn.Linear(2 * args["lstm_hidden"], args["out_features"])
 def forward(self, x1, x2):
x1 = self.embedding(x1.long() ).view(x1.shape[0], x1.shape[1], -1)
 x = torch.cat(( x1, x2), dim=2)
 x = self.lstm(x)[0][:, :x1.shape[1]]
x = self.fc(x)
 return x
def count_parameters(self):
return sum(p.numel() for p in self.parameters() if p.requires_grad )<find_best_params>
PL_PATH = ".. /input/covid-pl/"
def get_lr(optimizer):
for p in optimizer.param_groups:
return p["lr"]<compute_test_metric>
PL_PUBLIC = np.load(PL_PATH + 'pl_public.npy')
 PL_PRIVATE = np.load(PL_PATH + 'pl_private.npy' )
def MCRMSE(y_pred, y_true):
colwise_mse = torch.mean(( y_pred - y_true)**2, axis=1)
 return torch.mean(torch.sqrt(colwise_mse))<init_hyperparams>
for t, target in enumerate(TARGETS):
tgt = []
for i in range(len(public_df)) :
tgt.append(list(PL_PUBLIC[i, :SEQ_SCORED_PUBLIC, t]))
public_df[target] = tgt
tgt = []
for i in range(len(private_df)) :
tgt.append(list(PL_PRIVATE[i, :SEQ_SCORED_PRIVATE, t]))
private_df[target] = tgt
public_df['signal_to_noise'] = 1
private_df['signal_to_noise'] = 1
public_df = augment_data(public_df)
 private_df = augment_data(private_df )
TRAIN_MODE = False
args = {
"batch_size": 228,
"device": torch.device("cuda"),
"dtype": torch.float,
"embedding_dim": 41,
"epochs": 100,
"factor": 0.40876664932540246,
"folds": 5,
"in_features": 6,
"lr": 0.000514670300651814,
"lstm_dropout": 0.3,
"lstm_hidden": 765,
"lstm_layers": 2,
"min_lr": 1e-8,
"num_embeddings": len(token2int),
"out_features": 5,
"patience": 10,
"seed": 42,
"weight_decay": 0.23884714200145296,
}<load_pretrained>
def save_model_weights(model, filename, verbose=1, cp_folder=""):

 if verbose:
print(f"
-> Saving weights to {os.path.join(cp_folder, filename)}
")
 torch.save(model.state_dict() , os.path.join(cp_folder, filename))
def load_model_weights(model, filename, verbose=1, cp_folder=""):

 if verbose:
print(f"
-> Loading weights from {os.path.join(cp_folder,filename)}
")
 try:
model.load_state_dict(os.path.join(cp_folder, filename), strict=strict)
 except BaseException:
model.load_state_dict(
 torch.load(os.path.join(cp_folder, filename), map_location="cpu"),
strict=True, 
)
 return model
if TRAIN_MODE:
seed_everything(args["seed"])
 kfold = GroupKFold(n_splits=args["folds"])
 for fold,(train_idx, dev_idx)in enumerate(kfold.split(train, groups=train["cluster"])) :
train_loader = DataLoader(MyDataset(train.iloc[train_idx]), batch_size=args["batch_size"], shuffle=True, num_workers=3, pin_memory=True)
 dev_loader = DataLoader(MyDataset(train.iloc[dev_idx]), batch_size=args["batch_size"], shuffle=False, num_workers=3, pin_memory=True)
 model = MyModel(args ).to(device=args["device"], dtype=args["dtype"])
 optimizer = optim.AdamW(model.parameters() , lr=args["lr"], weight_decay=args["weight_decay"])
 scheduler = lr.ReduceLROnPlateau(optimizer, mode="min", factor=args["factor"], patience=args["patience"])
 pbar = tqdm(range(args["epochs"]))
best_loss = np.inf
for epoch in pbar:
model.train()
for x1, x2, y in train_loader:
x1 = x1.to(device=args["device"], dtype=torch.long)
 x2 = x2.to(device=args["device"], dtype=args["dtype"])
 y = y.to(device=args["device"], dtype=args["dtype"])
 y_pred = model(x1, x2)[:, :y.shape[1]]
loss = MCRMSE(y_pred, y)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
train_loss = 0
dev_loss = 0
with torch.no_grad() :
for batch,(x1, x2, y)in enumerate(train_loader):
x1 = x1.to(device=args["device"], dtype=torch.long)
 x2 = x2.to(device=args["device"], dtype=args["dtype"])
 y = y.to(device=args["device"], dtype=args["dtype"])
 y_pred = model(x1, x2)[:, :y.shape[1]]
loss = MCRMSE(y_pred, y)
 train_loss =(batch * train_loss + loss.item())/(batch + 1)
 for batch,(x1, x2, y)in enumerate(dev_loader):
x1 = x1.to(device=args["device"], dtype=torch.long)
 x2 = x2.to(device=args["device"], dtype=args["dtype"])
 y = y.to(device=args["device"], dtype=args["dtype"])
 y_pred = model(x1, x2)[:, :y.shape[1]]
loss = MCRMSE(y_pred, y)
 dev_loss =(batch * dev_loss + loss.item())/(batch + 1)
 scheduler.step(dev_loss)
 pbar.set_description("train: {:.5f} dev: {:.5f}".format(train_loss, dev_loss))
if dev_loss < best_loss:
best_loss = dev_loss
torch.save(model.state_dict() , "model_fold_{}.pth".format(fold))
if get_lr(optimizer)< args["min_lr"]:
break
print(best_loss )<create_dataframe>
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss
public_test = test[test["seq_length"] == 107].copy()
private_test = test[test["seq_length"] == 130].copy()<load_pretrained>
def create_loader(df, batch_size=64, is_test=False, shuffle=True):
if is_test:
shuffle = False
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=shuffle)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index] )
public_preds = np.zeros(( len(public_test), 107, len(targets)))
 private_preds = np.zeros(( len(private_test), 130, len(targets)))
 for fold in range(args["folds"]):
model = MyModel(args ).to(device=args["device"], dtype=args["dtype"])
 if TRAIN_MODE:
model.load_state_dict(torch.load("model_fold_{}.pth".format(fold)))
 else:
model.load_state_dict(torch.load(".. /input/openvaccine-trained/model_fold_{}.pth".format(fold)))
 model = model.eval()
public_test_loader = DataLoader(MyDataset(public_test, mode="test"), batch_size=args["batch_size"], shuffle=False, num_workers=3, pin_memory=True)
 private_test_loader = DataLoader(MyDataset(private_test, mode="test"), batch_size=args["batch_size"], shuffle=False, num_workers=3, pin_memory=True)
 with torch.no_grad() :
public_fold_preds = []
for batch,(x1, x2)in enumerate(public_test_loader):
x1 = x1.to(device=args["device"], dtype=torch.long)
 x2 = x2.to(device=args["device"], dtype=torch.long)
 y_pred = model(x1, x2 ).cpu().detach().numpy()
public_fold_preds.extend(y_pred)
 public_preds += np.array(public_fold_preds)/ args["folds"]
private_fold_preds = []
for batch,(x1, x2)in enumerate(private_test_loader):
x1 = x1.to(device=args["device"], dtype=torch.long)
 x2 = x2.to(device=args["device"], dtype=torch.long)
 y_pred = model(x1, x2 ).cpu().detach().numpy()
private_fold_preds.extend(y_pred)
 private_preds += np.array(private_fold_preds)/ args["folds"]<create_dataframe>
USE_FT = True
CNN_DROP = 0.1
ENC_DROP = 0.1
RNN_DROP = 0.3
LOGIT_DROP = 0.25
D = 256
preds = []
for df, p in [(public_test, public_preds),(private_test, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = p[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df["id_seqpos"] = [f"{uid}_{x}" for x in range(single_df.shape[0])]
preds.append(single_df)
 preds = pd.concat(preds )<save_to_csv>
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class Conv2dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv2dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv2d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int, out_dim=256):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, out_dim // 2, 3, padding=1)
 self.conv1 = Conv1dStack(out_dim // 2, out_dim // 4, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(out_dim // 4, out_dim // 8, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(out_dim // 8, out_dim // 8, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class BppAttn(nn.Module):
def __init__(self, in_channel: int, out_channel: int):
super(BppAttn, self ).__init__()
self.conv0 = Conv1dStack(in_channel, out_channel, 3, padding=1)
 self.bpp_conv = Conv2dStack(5, out_channel)
 def forward(self, x, bpp):
x = self.conv0(x)
 bpp = self.bpp_conv(bpp)
 x = torch.matmul(bpp, x.unsqueeze(-1))
return x.squeeze(-1)
 class PositionalEncoding(nn.Module):
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self ).__init__()
if ENC_DROP:
self.dropout = nn.Dropout(p=ENC_DROP)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
if ENC_DROP:
x = self.dropout(x)
 return x
class TransformerWrapper(nn.Module):
def __init__(self, dmodel=256, nhead=8, num_layers=2):
super(TransformerWrapper, self ).__init__()
self.pos_encoder = PositionalEncoding(256)
 encoder_layer = TransformerEncoderLayer(d_model=dmodel, nhead=nhead)
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.pos_emb = PositionalEncoding(dmodel)
 def flatten_parameters(self):
pass
def forward(self, x):
x = x.permute(( 1, 0, 2)).contiguous()
x = self.pos_emb(x)
 x = self.transformer_encoder(x)
 x = x.permute(( 1, 0, 2)).contiguous()
return x, None
class RnnLayers(nn.Module):
def __init__(self, dmodel, transformer_layers: int = 2):
super(RnnLayers, self ).__init__()
self.dropout = nn.Dropout(RNN_DROP)
 self.rnn0 = TransformerWrapper(dmodel, nhead=8, num_layers=transformer_layers)
 self.rnn1 = nn.LSTM(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 self.rnn2 = nn.GRU(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 def forward(self, x):
self.rnn0.flatten_parameters()
x, _ = self.rnn0(x)
 if self.rnn1 is not None:
self.rnn1.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn1(x)
 if self.rnn2 is not None:
self.rnn2.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn2(x)
 return x
class BaseAttnModel(nn.Module):
def __init__(self, transformer_layers=2, d=256):
super(BaseAttnModel, self ).__init__()
if USE_FT:
self.linear0 = nn.Linear(14 + 3, 1)
 self.seq_encoder_x = SeqEncoder(in_dim=18, out_dim=d)
 else:
self.linear0 = nn.Linear(14, 1)
 self.seq_encoder_x = SeqEncoder(in_dim=15, out_dim=d)
 self.attn = BppAttn(d, d//2)
 self.seq_encoder_bpp = SeqEncoder(in_dim=d//2, out_dim=d)
 self.seq = RnnLayers(d * 2, transformer_layers=transformer_layers)
 def forward(self, x, bpp):
bpp_features = get_bpp_feature(bpp[:, :, :, 0].float())
 if USE_FT:
x = torch.cat([x] + bpp_features, dim=-1)
 learned = self.linear0(x)
 x = torch.cat([x, learned], dim=-1)
 x = x.permute(0, 2, 1 ).contiguous().float()
bpp = bpp.permute([0, 3, 1, 2] ).contiguous().float()
x = self.seq_encoder_x(x)
 bpp = self.attn(x, bpp)
 bpp = self.seq_encoder_bpp(bpp)
 x = x.permute(0, 2, 1 ).contiguous()
bpp = bpp.permute(0, 2, 1 ).contiguous()
x = torch.cat([x, bpp], dim=2)
 x = self.seq(x)
 return x
class AEModel(nn.Module):
def __init__(self, transformer_layers=2):
super(AEModel, self ).__init__()
self.seq = BaseAttnModel(transformer_layers=transformer_layers, d=D)
 self.linear = nn.Sequential(
 nn.Linear(D * 2, 14),
nn.Sigmoid() , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = F.dropout(x, p=0.3)
 x = self.linear(x)
 return x
class FromAeModel(nn.Module):
def __init__(self, seq):
super(FromAeModel, self ).__init__()
self.seq = seq
self.linear = nn.Sequential(
 nn.Linear(D * 2, len(target_cols)) , 
)
 def forward(self, x, bpp, pred_len=68):
x = self.seq(x, bpp)
 x = self.linear(x)
 x = x[:, :pred_len]
return x
preds[["id_seqpos"] + targets].to_csv("submission.csv", index=False )<set_options>
PRETRAIN = False
warnings.filterwarnings('ignore')
<init_hyperparams>
features, _ = preprocess(train_df, True)
 features_tensor = torch.from_numpy(features)
 dataset0 = VacDataset(features_tensor, train_df, None)
 features, _ = preprocess(public_df, True)
 features_tensor = torch.from_numpy(features)
 dataset1 = VacDataset(features_tensor, public_df, None)
 features, _ = preprocess(private_df, True)
 features_tensor = torch.from_numpy(features)
 dataset2 = VacDataset(features_tensor, private_df, None )
SEED = 53
n_folds=5
debug=True
Window_features = True
model_name="GG"
epochs=100
BATCH_SIZE = 32
n_layers = 2
layers = ["GRU","GRU"]
hidden_dim = [128, 128]
dropout = [0.5, 0.5]
sp_dropout = 0.2
embed_dim = 250
num_hidden_units = 8
Cosine_Schedule = True
Rampup_decy_lr = False<define_variables>
BATCH_SIZE = 64
loader0 = torch.utils.data.DataLoader(dataset0, BATCH_SIZE, shuffle=True)
 loader1 = torch.utils.data.DataLoader(dataset1, BATCH_SIZE, shuffle=True)
 loader2 = torch.utils.data.DataLoader(dataset2, BATCH_SIZE, shuffle=True )
def seed_everything(seed=1234):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed_everything(SEED )<define_variables>
def learn_from_batch_ae(model, data):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(DEVICE), data["bpp"].to(DEVICE))
loss = F.binary_cross_entropy(out, target.to(DEVICE))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, start_epoch=0, start_it=0, log_path="./logs"):
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in range(start_epoch, end_epoch):
model.train()
losses = []
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'Epoch {epoch} \t loss={loss_m:.4f}')
 torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
window_columns = ['sequence','structure','predicted_loop_type']
categorical_features = ['sequence', 'structure', 'predicted_loop_type',]
cat_feature = len(categorical_features)
 if Window_features:
cat_feature += len(window_columns)
 numerical_features = ['BPPS_Max','BPPS_nb', 'BPPS_sum',
'positional_entropy',
'stems', 'interior_loops', 'multiloops',
'A_percent', 'G_percent','C_percent', 'U_percent',
'U-G', 'C-G', 'U-A', 'G-C', 'A-U', 'G-U',
'pair_map', 'pair_distance', ]
num_features = len(numerical_features)
 feature_cols = categorical_features + numerical_features
pred_col_names = ["pred_"+c_name for c_name in target_cols]
target_eval_col = ['reactivity','deg_Mg_pH10','deg_Mg_50C']
pred_eval_col = ["pred_"+c_name for c_name in target_eval_col]<load_from_csv>
set_seed(SEED )
data_dir = '/kaggle/input/stanford-covid-vaccine/'
fearure_data_path = '.. /input/openvaccine/'
train = pd.read_json(fearure_data_path+'train.json')
 test = pd.read_json(fearure_data_path+'test.json')
 sample_sub = pd.read_csv(data_dir + 'sample_submission.csv' )<feature_engineering>
if PRETRAIN:
model = AEModel()
model = model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
print(' -> Training with train data')
 res = train_ae(model, loader0, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 print('
-> Training with public data')
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 print('
-> Training with private data')
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 save_model_weights(model, CP_PATH + f'pretrained_model_{TODAY}.pt')
 print('
Done.' )
train[target_cols] = train[target_cols].applymap(lambda x: x[1:-1].split(", "))<filter>
CLASS_WEIGHT_5 = torch.from_numpy(np.array([1, 1, 1, 1, 1])).unsqueeze(0 ).cuda()
CLASS_WEIGHT_3 = torch.from_numpy(np.array([1, 1, 0, 1, 0])).unsqueeze(0 ).cuda()
train = train[train['signal_to_noise'] >= 0.5]<concatenate>
def mcrmse(truth, pred, verbose=0, scored_targets=[0, 1, 3], filtered=None, reduce=True):

 error =(truth - pred)** 2
error = error[:, :, scored_targets]
if filtered is not None:
error = np.array([error[i] for i, kept in enumerate(filtered)if kept])
 rmse = np.sqrt(error.mean(1))
if verbose:
for t, score in zip(scored_targets, rmse.mean(0)) :
print(f'Score for target "{TARGETS[t]}":\t {score:.4f}')
 if reduce:
return rmse.mean()
else:
return rmse.mean(-1)
 def MCRMSE(y_true, y_pred, class_weight):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse)* class_weight, dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise, class_weight):
loss = MCRMSE(target, predict, class_weight)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 weight = torch.clamp(weight, 0.01, 1000)
 loss =(loss * weight ).mean()
return loss
def pair_feature(row):
arr = list(row)
 its = [iter(['_']+arr[:]),iter(arr[1:]+['_'])]
list_touple = list(zip(*its))
return list(map("".join,list_touple))<feature_engineering>
def learn_from_batch(model, data, optimizer, lr_scheduler, class_weight, pred_len=68):
optimizer.zero_grad()
out = model(
 data["sequence"].to(DEVICE),
data["bpp"].to(DEVICE),
pred_len=pred_len, 
)
 signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(
 out,
data["label"].to(DEVICE),
signal_to_noise.to(DEVICE),
class_weight, 
)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, class_weight):
model.eval()
loss_list = []
mcrmses = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(
 data["sequence"].to(DEVICE),
data["bpp"].to(DEVICE),
pred_len=68, 
)
 mcrmse_ = mcrmse(data["label"].numpy() , y.cpu().numpy() , filtered=data["signal_to_noise"] > 1, reduce=False)
 mcrmses += list(mcrmse_)
 loss = sn_mcrmse_loss(
 y,
data["label"].to(DEVICE),
data["signal_to_noise"].to(DEVICE),
class_weight 
)
 loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcrmse=np.mean(mcrmses))
def preprocess_categorical_inputs(df, cols=categorical_features,Window_features=Window_features):
if Window_features:
for c in window_columns:
df["pair_"+c] = df[c].apply(pair_feature)
 cols.append("pair_"+c)
 cols = list(set(cols))
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
 )<categorify>
def predict(model, loader, pred_len=68):

 model.eval()
preds = np.empty(( 0, pred_len, NUM_TARGETS))
with torch.no_grad() :
for batch in loader:
y_pred = model(
 batch["sequence"].cuda() ,
batch["bpp"].cuda() ,
pred_len=pred_len,
).detach()
preds = np.concatenate([preds, y_pred.cpu().numpy() ])
 return preds
def preprocess_numerical_inputs(df, cols=numerical_features):
return np.transpose(
 np.array(
 df[cols].values.tolist() 
), 
(0, 2, 1)
 )<define_variables>
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, swa_first_epoch=40, class_weight=None, pred_len=68):
it = 0
for epoch in range(epochs):
t0 = time.time()
print(f"Epoch {epoch+1}/{epochs}", end='\t')
 model.train()
losses = []
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, class_weight, pred_len=pred_len)
 loss_v = loss.item()
losses.append(loss_v)
 it += 1
if epoch + 1 >= swa_first_epoch:
optimizer.update_swa()
optimizer.swap_swa_sgd()
eval_result = evaluate(model, valid_data, class_weight)
 eval_loss = eval_result["loss"]
if epoch + 1 >= swa_first_epoch and epoch < epochs - 1:
optimizer.swap_swa_sgd()
dt = time.time() - t0
lr = lr_scheduler.get_last_lr() [0] if lr_scheduler else 1e-3
print(f't={dt:.1f}s\t lr={lr:.1e}\tloss={np.mean(losses):.4f}', end='\t')
 print(f"val_loss={eval_loss:.4f} \t val_mcrmse={eval_result['mcrmse']:.4f}" )
token_list = list("().ACGUESHBXIM")
 if Window_features:
comb = combinations_with_replacement(list('_().ACGUESHBXIM'*2), 2)
 token_list += list(set(list(map("".join,comb))))
token2int = {x:i for i, x in enumerate(list(set(token_list)))}
print("token_list Size :",len(token_list))
train_inputs_all_cat = preprocess_categorical_inputs(train,cols=categorical_features)
 train_inputs_all_num = preprocess_numerical_inputs(train,cols=numerical_features)
 train_labels_all = np.array(train[target_cols].values.tolist() ,dtype =np.float32 ).transpose(( 0, 2, 1))
print("Train categorical Features Shape : ",train_inputs_all_cat.shape)
 print("Train numerical Features Shape : ",train_inputs_all_num.shape)
 print("Train labels Shape : ",train_labels_all.shape )<compute_test_metric>
EPOCHS_1 = 30
EPOCHS_2 = 10
EPOCHS_3 = 10
EPOCHS_4 = 5
SWA_FIRST_EPOCH = 0
WARMUP_PROP = 0.05
K = 5
BATCH_SIZE = 32
LR = 5e-4
LOAD = True
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true[:,:,:3] - y_pred[:,:,:3]), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )<choose_model_class>
samples = train_df.copy().drop('score', axis=1)
 ids = samples.reset_index() ["id"]
set_seed(SEED )
def get_lr_callback(batch_size=8):
lr_start = 0.00001
lr_max = 0.004
lr_min = 0.00005
lr_ramp_ep = 45
lr_sus_ep = 2
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
gkf = GroupKFold(n_splits=K)
 splits = list(gkf.split(X=samples, groups=groups))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))<choose_model_class>
scores = []
pred_oof = np.zeros(( len(samples), 68, NUM_TARGETS))
for fold,(train_index, test_index)in enumerate(splits):
print(f"
------------- Fold {fold + 1}/{K} -------------
")
 set_seed(SEED)
 df_train = samples.loc[train_index].reset_index()
df_train = augment_data(df_train)
 df_val = samples.loc[test_index].reset_index()
df_val_aug = augment_data(df_val.copy() , concat=False)
 train_loader = create_loader(df_train, BATCH_SIZE)
 public_loader = create_loader(public_df, BATCH_SIZE)
 private_loader = create_loader(private_df, BATCH_SIZE)
 valid_loader = create_loader(df_val, BATCH_SIZE, shuffle=False)
 valid_loader_aug = create_loader(df_val_aug, BATCH_SIZE, shuffle=False)
 ae_model = AEModel()
if LOAD:
load_model_weights(ae_model, PRETRAINED_PATH)
 model = FromAeModel(ae_model.seq)
 model = model.to(DEVICE)
 model.zero_grad()
model.train()
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 if SWA_FIRST_EPOCH < EPOCHS_4:
optimizer = SWA(optimizer)
 s1 = EPOCHS_1 * len(train_loader)
 s2 = EPOCHS_2 * len(public_loader)
 s3 = EPOCHS_3 * len(private_loader)
 s4 = EPOCHS_4 * len(train_loader)
 num_training_steps = int(s1 + s2 + s3 + s4)
 num_warmup_steps = int(EPOCHS_1 * len(train_loader)* WARMUP_PROP)
 lr_scheduler = get_linear_schedule_with_warmup(
 optimizer, num_warmup_steps, num_training_steps 
)
 print('
-> Training with train data
')
 train(
 model, train_loader, valid_loader, optimizer, lr_scheduler, pred_len=68,
epochs=EPOCHS_1, swa_first_epoch=100, class_weight=CLASS_WEIGHT_5)
 print('
-> Training with public data
')
 train(
 model, public_loader, valid_loader, optimizer, lr_scheduler, pred_len=SEQ_SCORED_PUBLIC,
epochs=EPOCHS_2, swa_first_epoch=100, class_weight=CLASS_WEIGHT_3 
)
 print('
-> Training with private data
')
 train(
 model, private_loader, valid_loader, optimizer, lr_scheduler, pred_len=SEQ_SCORED_PRIVATE,
epochs=EPOCHS_3, swa_first_epoch=100, class_weight=CLASS_WEIGHT_3 
)
 print('
-> Retraining with train data
')
 train(
 model, train_loader, valid_loader, optimizer, lr_scheduler, pred_len=68,
epochs=EPOCHS_4, swa_first_epoch=SWA_FIRST_EPOCH, class_weight=CLASS_WEIGHT_3 
)
 y = np.array(df_val[TARGETS].values.tolist() ).transpose(( 0, 2, 1))
pred_val_ = predict(model, valid_loader)
 pred_val_aug = predict(model, valid_loader_aug)
 pred_val =(0.5 * pred_val_ + 0.5 * pred_val_aug)
 pred_oof[test_index] = pred_val
score = mcrmse(y, pred_val, verbose=0, scored_targets=[0, 1, 3], filtered=df_val["signal_to_noise"] > 1)
 scores.append(score)
 print(f"
-> Scored {score:.4f} with TTA")
 save_model_weights(model, f'model_{fold}.pt', verbose=0, cp_folder=CP_PATH)
 del model
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(
 L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer='orthogonal')
 )<choose_model_class>
BATCH_SIZE = 64
TTA = True
def build_model(embed_size,
seq_len = 107,
pred_len = 68,
dropout = dropout,
sp_dropout = sp_dropout,
num_features = num_features,
num_hidden_units = num_hidden_units,
embed_dim = embed_dim,
layers = layers,
hidden_dim = hidden_dim,
n_layers = n_layers,
cat_feature = cat_feature):
inputs = L.Input(shape=(seq_len, cat_feature),name='category_input')
 embed = L.Embedding(input_dim=embed_size, output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.SpatialDropout1D(sp_dropout )(reshaped)
 reshaped_conv = tf.keras.layers.Conv1D(filters=512, kernel_size=3,strides=1, padding='same', activation='elu' )(reshaped)
 numerical_input = L.Input(shape=(seq_len, num_features), name='numeric_input')
 hidden = L.concatenate([reshaped_conv, numerical_input])
 hidden_1 = tf.keras.layers.Conv1D(filters=256, kernel_size=4,strides=1, padding='same', activation='elu' )(hidden)
 hidden = gru_layer(128, 0.5 )(hidden_1)
 hidden = L.concatenate([hidden, hidden_1])
 for x in range(n_layers):
if layers[x] == "GRU":
hidden = gru_layer(hidden_dim[x], dropout[x] )(hidden)
 else:
hidden = lstm_layer(hidden_dim[x], dropout[x] )(hidden)
 hidden = L.concatenate([hidden, hidden_1])
 truncated = hidden[:, :pred_len]
out = L.Dense(5 )(truncated)
 model = tf.keras.Model(inputs=[inputs] + [numerical_input], outputs=out)
 adam = tf.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead = tfa.optimizers.Lookahead(adam, sync_period=6)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6)
 model.compile(optimizer=radam, loss=MCRMSE)
 return model<concatenate>
test_df = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = test_df[test_df["seq_length"] == SEQ_LEN_PUBLIC].reset_index(drop=True)
 private_df = test_df[test_df["seq_length"] == SEQ_LEN_PRIVATE].reset_index(drop=True)
 pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 public_df_aug = augment_data(public_df.copy().drop('score', axis=1), concat=False)
 pub_loader_aug = create_loader(public_df_aug, BATCH_SIZE, is_test=True)
 private_df_aug = augment_data(private_df.copy().drop('score', axis=1), concat=False)
 pri_loader_aug = create_loader(private_df_aug, BATCH_SIZE, is_test=True )
def get_stratify_group(row):
snf = row['SN_filter']
snr = row['signal_to_noise']
cnt = row['cnt']
id_ = row['id']
structure = row['structure']
if snf == 0:
if snr<0:
snr_c = 0
elif 0<= snr < 2:
snr_c = 1
elif 2<= snr < 4:
snr_c = 2
elif 4<= snr < 5.5:
snr_c = 3
elif 5.5<= snr < 10:
snr_c = 4
elif snr >= 10:
snr_c = 5
else:
if snr<0:
snr_c = 6
elif 0<= snr < 1:
snr_c = 7
elif 1<= snr < 2:
snr_c = 8
elif 2<= snr < 3:
snr_c = 9
elif 3<= snr < 4:
snr_c = 10
elif 4<= snr < 5:
snr_c = 11
elif 5<= snr < 6:
snr_c = 12
elif 6<= snr < 7:
snr_c = 13
elif 7<= snr < 8:
snr_c = 14
elif 8<= snr < 9:
snr_c = 15
elif 9<= snr < 10:
snr_c = 15
elif snr >= 10:
snr_c = 16
return '{}_{}'.format(id_,snr_c)
 train['stratify_group'] = train.apply(get_stratify_group, axis=1)
 train['stratify_group'] = train['stratify_group'].astype('category' ).cat.codes
skf = StratifiedKFold(n_folds, shuffle=True, random_state=SEED)
 gkf = GroupKFold(n_splits=n_folds)
 fig, ax = plt.subplots(n_folds,3,figsize=(20,5*n_folds))
for Fold,(train_index, val_index)in enumerate(gkf.split(train_inputs_all_cat, groups=train['stratify_group'])) :
print(Fore.YELLOW);print('
train_data = train.iloc[train_index]
val_data = train.iloc[val_index]
print("Augmented data Present in Val Data : ",len(val_data[val_data['cnt'] != 1]))
print("Augmented data Present in Train Data : ",len(train_data[train_data['cnt'] != 1]))
val_data = val_data[val_data['cnt'] == 1]
print("Data Lekage : ",len(val_data[val_data['id'].isin(train_data['id'])]))
print("number of Train Data points : ",len(train_data))
print("number of val_data Data points : ",len(val_data))
print("number of unique Structure in Train data : ", len(train_data.structure.unique()))
print("number of unique Structure in val data : ",len(val_data.structure.unique()), val_data.structure.value_counts() [:5].values)
 print("Train SN_Filter == 1 : ", len(train_data[train_data['SN_filter']==1]))
print("val_data SN_Filter == 1 : ", len(val_data[val_data['SN_filter']==1]))
print("Train SN_Filter == 0 : ", len(train_data[train_data['SN_filter']==0]))
print("val_data SN_Filter == 0 : ", len(val_data[val_data['SN_filter']==0]))
print("Unique ID :",len(train_data.id.unique()))
sns.kdeplot(train[train['SN_filter']==0]['signal_to_noise'],ax=ax[Fold][0],color="Red",label='Train All')
 sns.kdeplot(train_data[train_data['SN_filter']==0]['signal_to_noise'],ax=ax[Fold][0],color="Blue",label='Train')
 sns.kdeplot(val_data[val_data['SN_filter']==0]['signal_to_noise'],ax=ax[Fold][0],color="Green",label='Validation')
 ax[Fold][0].set_title(f'Fold : {Fold+1} Signal/Noise & SN_filter == 0')
 sns.kdeplot(train[train['SN_filter']==1]['signal_to_noise'],ax=ax[Fold][1],color="Red",label='Train All')
 sns.kdeplot(train_data[train_data['SN_filter']==1]['signal_to_noise'],ax=ax[Fold][1],color="Blue",label='Train')
 sns.kdeplot(val_data[val_data['SN_filter']==1]['signal_to_noise'],ax=ax[Fold][1],color="Green",label='Validation')
 ax[Fold][1].set_title(f'Fold : {Fold+1} Signal/Noise & SN_filter == 1')
 sns.kdeplot(train['signal_to_noise'],ax=ax[Fold][2],color="Red",label='Train All')
 sns.kdeplot(train_data['signal_to_noise'],ax=ax[Fold][2],color="Blue",label='Train')
 sns.kdeplot(val_data['signal_to_noise'],ax=ax[Fold][2],color="Green",label='Validation')
 ax[Fold][2].set_title(f'Fold : {Fold+1} Signal/Noise')
 plt.show() 
<create_dataframe>
pred_df_list = []
pred_public = np.zeros(( len(public_df), 107, NUM_TARGETS))
pred_private = np.zeros(( len(private_df), SEQ_LEN_PRIVATE, NUM_TARGETS))
for fold in range(K):
print(f"
------------- Fold {fold + 1}/{K} -------------
")
 model_load_path = CP_PATH + f"model_{fold}.pt"
print(f' -> Loading weights from {model_load_path}
')
 ae_model0 = AEModel()
model_pub = FromAeModel(seq=ae_model0.seq)
 model_pub = model_pub.to(DEVICE)
 ae_model1 = AEModel()
model_pri = FromAeModel(seq=ae_model1.seq)
 model_pri = model_pri.to(DEVICE)
 state_dict = torch.load(model_load_path, map_location=DEVICE)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
pred_public += predict(model_pub, pub_loader, pred_len=107)/ K
pred_private += predict(model_pri, pri_loader, pred_len=130)/ K
if TTA:
pred_public += predict(model_pub, pub_loader_aug, pred_len=107)/ K
pred_private += predict(model_pri, pri_loader_aug, pred_len=130)/ K
if TTA:
pred_public *= 0.5
pred_private *= 0.5
submission = pd.DataFrame(index=sample_sub.index, columns=target_cols ).fillna(0)
 val_losses = []
historys = []
oof_preds_all = []
stacking_pred_all = []
kf = KFold(n_folds, shuffle=True, random_state=SEED)
 skf = StratifiedKFold(n_folds, shuffle=True, random_state=SEED)
 gkf = GroupKFold(n_splits=n_folds)
 for Fold,(train_index, val_index)in enumerate(gkf.split(train_inputs_all_cat, groups=train['stratify_group'])) :
print(Fore.YELLOW);print('
print(f"|| Batch_size: {BATCH_SIZE}
|| n_layers: {n_layers}
|| embed_dim: {embed_dim}")
 print(f"|| cat_feature: {cat_feature}
|| num_features: {num_features}")
 print(f"|| layers : {layers}
|| hidden_dim: {hidden_dim}
|| dropout: {dropout}
|| sp_dropout: {sp_dropout}")
 train_data = train.iloc[train_index]
val_data = train.iloc[val_index]
print("|| number Augmented data Present in Val Data : ",len(val_data[val_data['cnt'] != 1]))
print("|| number Augmented data Present in Train Data : ",len(train_data[train_data['cnt'] != 1]))
print("|| Data Lekage : ",len(val_data[val_data['id'].isin(train_data['id'])]))
val_data = val_data[val_data['cnt'] == 1]
model_train = build_model(embed_size=len(token_list))
model_short = build_model(embed_size=len(token_list),seq_len=107, pred_len=107)
 model_long = build_model(embed_size=len(token_list),seq_len=130, pred_len=130)
 train_inputs_cat = preprocess_categorical_inputs(train_data,cols=categorical_features)
 train_inputs_num = preprocess_numerical_inputs(train_data,cols=numerical_features)
 train_labels = np.array(train_data[target_cols].values.tolist() ,dtype =np.float32 ).transpose(( 0, 2, 1))
val_inputs_cat = preprocess_categorical_inputs(val_data,cols=categorical_features)
 val_inputs_num = preprocess_numerical_inputs(val_data,cols=numerical_features)
 val_labels = np.array(val_data[target_cols].values.tolist() ,dtype =np.float32 ).transpose(( 0, 2, 1))
csv_logger = tf.keras.callbacks.CSVLogger(f'Fold_{Fold}_log.csv', separator=',', append=False)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{model_name}_Fold_{Fold}.h5',
monitor='val_loss',
verbose=0,
mode='min',
save_freq='epoch')
 if Cosine_Schedule:
lr_schedule= get_cosine_schedule_with_warmup(lr=0.001, num_warmup_steps=20, num_training_steps=epochs)
 elif Rampup_decy_lr :
lr_schedule = get_lr_callback(BATCH_SIZE)
 else:
lr_schedule = tf.keras.callbacks.ReduceLROnPlateau()
history = model_train.fit(
 {'numeric_input': train_inputs_num,
'category_input': train_inputs_cat} , train_labels,
validation_data=({'numeric_input': val_inputs_num,
'category_input': val_inputs_cat}
,val_labels),
batch_size=BATCH_SIZE,
epochs=epochs,
callbacks=[lr_schedule, checkpoint, csv_logger,lr_schedule],
verbose=1 if debug else 0 
)
 print("Min Validation Loss : ", min(history.history['val_loss']))
print("Min Validation Epoch : ",np.argmin(history.history['val_loss'])+1)
 val_losses.append(min(history.history['val_loss']))
historys.append(history)
 model_short.load_weights(f'{model_name}_Fold_{Fold}.h5')
 model_long.load_weights(f'{model_name}_Fold_{Fold}.h5')
 public_preds = model_short.predict({'numeric_input': public_inputs_num,
'category_input': public_inputs_cat})
 private_preds = model_long.predict({'numeric_input': private_inputs_num,
'category_input': private_inputs_cat})
 oof_preds = model_train.predict({'numeric_input': val_inputs_num,
'category_input': val_inputs_cat})
 stacking_pred = model_short.predict({'numeric_input': val_inputs_num,
'category_input': val_inputs_cat})
 preds_model = []
for df, preds in [(public_df, public_preds),(private_df, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model.append(single_df)
 preds_model_df = pd.concat(preds_model)
 preds_model_df = preds_model_df.groupby(['id_seqpos'],as_index=True ).mean()
submission[target_cols] += preds_model_df[target_cols].values / n_folds
for df, preds in [(val_data, oof_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_label = val_labels[i]
single_label_df = pd.DataFrame(single_label, columns=target_cols)
 single_label_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_label_df.shape[0])]
single_label_df['id'] = [f'{uid}' for x in range(single_label_df.shape[0])]
single_label_df['s_id'] = [x for x in range(single_label_df.shape[0])]
single_df = pd.DataFrame(single_pred, columns=pred_col_names)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df = pd.merge(single_label_df,single_df, on="id_seqpos", how="left")
 oof_preds_all.append(single_df)
 for df, preds in [(val_data, stacking_pred)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_col_names)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['id'] = [uid for x in range(single_df.shape[0])]
stacking_pred_all.append(single_df)
 history_data = pd.read_csv(f'Fold_{Fold}_log.csv')
 EPOCHS = len(history_data['epoch'])
 history = pd.DataFrame({'history':history_data.to_dict('list')})
 fig = plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS),history.history['lr'],'-',label='Learning Rate',color='
x = np.argmax(history.history['lr']); y = np.max(history.history['lr'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Learning Rate',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=1)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 fig.text(s=f"Model Name : {model_name}" , x=0.5, y=1.08, fontsize=18, ha='center', va='center',color="green")
 fig.text(s=f"|| Fold : {Fold+1} | Batch_size: {BATCH_SIZE} | num_features: {num_features} | cat_feature: {cat_feature} |n_layers: {n_layers} | embed_dim: {embed_dim} ||", x=0.5, y=1.0, fontsize=15, ha='center', va='center',color="red")
 fig.text(s=f"|| layers : {layers} | hidden_dim: {hidden_dim} | dropout: {dropout} | sp_dropout: {sp_dropout} ||", x=0.5, y=0.92, fontsize=15, ha='center', va='center',color="blue")
 plt.legend(loc=3)
 plt.savefig(f'Fold_{Fold+1}.png', bbox_inches='tight')
 plt.show()
submission["id_seqpos"] = preds_model_df.index
submission = pd.merge(sample_sub["id_seqpos"], submission, on="id_seqpos", how="left")
 OOF = pd.concat(oof_preds_all)
 stacking_df = pd.concat(stacking_pred_all )<save_to_csv>
def pred_to_sub(df_test, pred_public, pred_private):
sub_public = df_test[df_test['seq_scored'] == SEQ_SCORED_PUBLIC][['id']].reset_index(drop=True)
 sub_private = df_test[df_test['seq_scored'] == SEQ_SCORED_PRIVATE][['id']].reset_index(drop=True)
 test_preds = []
for sub, pred in [(sub_public, pred_public),(sub_private, pred_private)]:
for i, seq in enumerate(sub.id):
single_pred = pred[i]
single_df = pd.DataFrame(single_pred, columns=TARGETS)
 single_df['id_seqpos'] = [f'{seq}_{x}' for x in range(single_df.shape[0])]
test_preds.append(single_df)
 return pd.concat(test_preds)[['id_seqpos'] + TARGETS]
OOF = OOF.groupby(['id_seqpos','id','s_id'],as_index=False ).mean()
OOF = OOF.sort_values(['id','s_id'],ascending=[True, True])
 OOF_score = MCRMSE(np.expand_dims(OOF[target_eval_col].values, axis=0), np.expand_dims(OOF[pred_eval_col].values, axis=0)).numpy() [0]
print("Overall OOF Score :",OOF_score)
 OOF.to_csv('OOf.csv',index=True)
 OOF.head()<save_to_csv>
sub = pred_to_sub(test_df, pred_public, pred_private )
OOF_filter_1 = pd.merge(train[['SN_filter','id']],OOF,on='id')
 OOF_filter_1 = OOF_filter_1.groupby(['id_seqpos','id'],as_index=False ).mean()
OOF_filter_1 = OOF_filter_1.sort_values(['id','s_id'],ascending=[True, True])
 OOF_filter_1 = OOF_filter_1[OOF_filter_1['SN_filter'] == 1]
OOF_filter_1_score = MCRMSE(np.expand_dims(OOF_filter_1[target_eval_col].values, axis=0), np.expand_dims(OOF_filter_1[pred_eval_col].values, axis=0)).numpy() [0]
print("OOF_public Score :",OOF_filter_1_score)
 OOF_filter_1.to_csv('OOF_filter_1.csv',index=False)
 OOF_filter_1.head()<save_to_csv>
score = np.mean(scores)
 score
<save_to_csv><EOS> 
print(f'Saving submission to "{TODAY}_{score:.4f}_pl.csv"')
 sub.to_csv(f"{TODAY}_{score:.4f}_pl.csv", index=False)
 np.save(f"oof_{TODAY}_{score:.4f}_pl.npy", pred_oof)
 sub.head()