Kaggle competition: Mechanisms of Action (MoA) Prediction
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
sequence_token2int = {x:i for i, x in enumerate('AGUC')}
structure_token2int = {
'.': 0,
'(': 1,
')': 2,
}
loop_token2int = {x:i for i, x in enumerate('SMIBHEX')}
token2int_map = {
"sequence": sequence_token2int,
"structure": structure_token2int,
"predicted_loop_type": loop_token2int
}
sequence_columns = ["sequence", "structure", "predicted_loop_type"]
def to_seq(df):
return np.transpose(
 np.array(
 df[sequence_columns] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 train = to_seq(train_df)
 public = to_seq(public_df)
 private = to_seq(private_df)
 train.shape, public.shape, private.shape<categorify>
warnings.filterwarnings('ignore' )
def to_one_hot(df):
temp = np.transpose(
 np.array([
df[col] 
.apply(lambda seq: [token2int_map[col][x] for x in seq])
.values 
.tolist()
for col in sequence_columns
]), 
(1, 2, 0)
)
 ohe_1 = tf.keras.utils.to_categorical(temp[:,:,0], 4)
 ohe_2 = tf.keras.utils.to_categorical(temp[:,:,1], 3)
 ohe_3 = tf.keras.utils.to_categorical(temp[:,:,2], 7)
 return np.concatenate([ohe_1, ohe_2, ohe_3], axis=2)
 train_ohe = to_one_hot(train_df)
 public_ohe = to_one_hot(public_df)
 private_ohe = to_one_hot(private_df)
 train_ohe.shape, public_ohe.shape, private_ohe.shape<concatenate>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
def get_adjacency_matrix(inps):
As = []
for row in range(0, inps.shape[0]):
A = np.zeros(( inps.shape[1], inps.shape[1]))
stack = []
opened_so_far = []
for seqpos in range(0, inps.shape[1]):
if inps[row, seqpos, 1] == 0:
stack.append(seqpos)
 opened_so_far.append(seqpos)
 elif inps[row, seqpos, 1] == 1:
openpos = stack.pop()
A[openpos, seqpos] = 1
A[seqpos, openpos] = 1
As.append(A)
 return np.array(As)
 train_adj = get_adjacency_matrix(train)
 public_adj = get_adjacency_matrix(public)
 private_adj = get_adjacency_matrix(private)
 train_adj.shape, public_adj.shape, private_adj.shape<categorify>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
def get_bpps(mRNA_ids):
bpps = []
for mRNA_id in tqdm(mRNA_ids):
bpps.append(
 np.load(f"{bpps_path}/{mRNA_id}.npy"), 
)
 return np.array(bpps)
 train_bpps = get_bpps(train_df.id.values)
 public_bpps = get_bpps(public_df.id.values)
 private_bpps = get_bpps(private_df.id.values)
 train_bpps.shape, public_bpps.shape, private_bpps.shape<define_variables>
def seed_everything(seed=321):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=321 )
train_bpps_stats = [train_bpps.mean(axis=2), train_bpps.max(axis=2)]
public_bpps_stats = [public_bpps.mean(axis=2), public_bpps.max(axis=2)]
private_bpps_stats = [private_bpps.mean(axis=2), private_bpps.max(axis=2)]<concatenate>
n_comp = 29
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
train_bpps_stats = np.concatenate([stats[:,:,None] for stats in train_bpps_stats], axis=2)
 public_bpps_stats = np.concatenate([stats[:,:,None] for stats in public_bpps_stats], axis=2)
 private_bpps_stats = np.concatenate([stats[:,:,None] for stats in private_bpps_stats], axis=2)
 train_bpps_stats.shape, public_bpps_stats.shape, private_bpps_stats.shape<compute_test_metric>
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features
scored_seq_length = 68
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred):
score = 0
for i in range(y_actual.shape[2]):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ y_actual.shape[2]
return score
def build_model(input_seq_len=107, output_seq_len=scored_seq_length):
def _bi_gru_block(x, hidden_dim, dropout):
gru = L.Bidirectional(
 L.GRU(hidden_dim,
dropout=dropout,
return_sequences=True, 
),
)(x)
 return gru
def _conv_block(x, adj_m, bpp_m, conv_filters, graph_channels):
conv = L.Conv1D(
 conv_filters, 5,
padding='same',
activation='tanh',
)(x)
 gcn_1 = GraphConv(
 graph_channels,
activation='tanh',
)([conv, adj_m])
 gcn_2 = GraphConv(
 graph_channels,
activation='tanh',
)([conv, bpp_m])
 conv = L.Concatenate()([conv, gcn_1, gcn_2])
 conv = L.Activation("relu" )(conv)
 conv = L.SpatialDropout1D(0.1 )(conv)
 return conv
one_hot_encoding_inputs = L.Input(shape=(input_seq_len, 14), name="onehot")
 adj_matrix_inputs = L.Input(( input_seq_len, input_seq_len), name="adjmatrix")
 base_pair_proba_inputs = L.Input(( input_seq_len, input_seq_len), name="pairproba")
 base_pair_proba_stats_inputs = L.Input(shape=(input_seq_len, 2), name="pairprobastats")
 merged_inputs = L.Concatenate()([one_hot_encoding_inputs, base_pair_proba_stats_inputs])
 hidden = _conv_block(merged_inputs, adj_matrix_inputs, base_pair_proba_inputs, 512, 80)
 hidden = _bi_gru_block(hidden, 256, 0.5)
 hidden = _conv_block(hidden, adj_matrix_inputs, base_pair_proba_inputs, 512, 80)
 hidden = _bi_gru_block(hidden, 256, 0.5)
 out = hidden[:, :output_seq_len]
out = L.Dense(5, activation='linear' )(out)
 model = tf.keras.Model(
 inputs=[
one_hot_encoding_inputs,
adj_matrix_inputs,
base_pair_proba_inputs,
base_pair_proba_stats_inputs,
],
outputs=out, 
)
 return model
model = build_model()
plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True )<split>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
split_results = train_test_split(
 train_ohe,
train_adj,
train_bpps,
train_bpps_stats,
train_y,
train_df.signal_to_noise,
train_df.SN_filter,
test_size=0.1,
random_state=42, 
)
 [a.shape for a in split_results]<split>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
trn_ohe, val_ohe, trn_adj, val_adj, trn_bpps, val_bpps, trn_bpps_stats, val_bpps_stats, trn_y, val_y, trn_snr, val_snr, trn_snf, val_snf = split_results<choose_model_class>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=5)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )
model = build_model()
model.compile(tf.keras.optimizers.Adam() , loss=mcrmse )<define_variables>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
trn_inputs = [trn_ohe, trn_adj, trn_bpps, trn_bpps_stats]
val_inputs = [val_ohe, val_adj, val_bpps, val_bpps_stats]<prepare_x_and_y>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
val_mask = np.where(( val_snf==1))
val_inputs = [val_input[val_mask] for val_input in val_inputs]
val_y = val_y[val_mask]<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.3)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
sample_weight = np.log(trn_snr+1.11)/2<train_model>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
history = model.fit(
 trn_inputs, trn_y,
validation_data =(val_inputs, val_y),
batch_size=64,
epochs=300,
sample_weight=sample_weight,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(verbose=1, monitor='val_loss'),
tf.keras.callbacks.ModelCheckpoint(f'model.h5',save_best_only=True, verbose=0, monitor='val_loss'),
tf.keras.callbacks.EarlyStopping(
 patience=20,
monitor='val_loss',
verbose=0,
mode="auto",
baseline=None,
restore_best_weights=True, 
),
],
verbose=2 
)
 print(f"Min validation loss history={min(history.history['val_loss'])}" )<load_pretrained>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
model.load_weights(f'model.h5' )<predict_on_test>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 25
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
val_preds = model.predict(val_inputs)
 tf.reduce_mean(mcrmse(val_y, val_preds))<load_pretrained>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
model_public = build_model(107, 107)
 model_private = build_model(130, 130)
 model_public.load_weights(f'model.h5')
 model_private.load_weights(f'model.h5' )<define_variables>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
public_inputs = [public_ohe, public_adj, public_bpps, public_bpps_stats,]
private_inputs = [private_ohe, private_adj, private_bpps, private_bpps_stats,]<predict_on_test>
SEED = [456]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
test_preds = [model_public.predict(public_inputs), model_private.predict(private_inputs)]
test_dfs = [public_df, private_df]
test_preds[0].shape, test_preds[1].shape<prepare_output>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
preds_ls = []
for df, preds in zip(test_dfs, test_preds):
for i, uid in tqdm(enumerate(df.id)) :
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
test_df.shape, preds_df.shape<save_to_csv>
sub= sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )<import_modules>
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from tqdm.notebook import tqdm
from time import time
!pip install nyaggle 
<define_variables>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
TOKEN2INT = {x: i for i, x in enumerate('().ACGUBEHIMSX')}
PRED_COLS_SCORED = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']
PRED_COLS = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
DATA_DIR = ".. /input/stanford-covid-vaccine/"
REPLACE_DATA_PATH = ".. /input/eternafold/eternafold_mfe.csv"
PRIMARY_BPPS_DIR = ".. /input/eternafold/bpps/"
SECONDARY_BPPS_DIR = ".. /input/bpps-by-viennat70/"
NFOLDS = 7
BATCH_SIZE = 64
TRAIN_EPOCHS = 140<set_options>
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )<statistical_test>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
class SELayer(nn.Module):
def __init__(self, channel, reduction=16):
super(SELayer, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool1d(1)
 self.fc = nn.Sequential(
 nn.Linear(channel, channel // reduction, bias=False),
nn.ReLU(inplace=True),
nn.Linear(channel // reduction, channel, bias=False),
nn.Sigmoid() 
)
 def forward(self, x):
b, c, _ = x.size()
y = self.avg_pool(x ).view(b, c)
 y = self.fc(y ).view(b, c, 1)
 return x * y.expand_as(x)
 class Conv(nn.Module):
def __init__(self, d_in, d_out, kernel_size, dropout=0.1):
super().__init__()
self.conv = nn.Conv1d(d_in, d_out, kernel_size=kernel_size, padding=kernel_size // 2)
 self.bn = nn.BatchNorm1d(d_out)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(dropout)
 def forward(self, src):
return self.dropout(self.relu(self.bn(self.conv(src))))
class ResidualGraphAttention(nn.Module):
def __init__(self, d_model, kernel_size, dropout):
super().__init__()
self.conv1 = Conv(d_model, d_model, kernel_size=kernel_size, dropout=dropout)
 self.conv2 = Conv(d_model, d_model, kernel_size=kernel_size, dropout=dropout)
 self.relu = nn.ReLU()
def forward(self, src, attn):
h = self.conv2(self.conv1(torch.bmm(src, attn)))
 return self.relu(src + h)
 class SEResidual(nn.Module):
def __init__(self, d_model, kernel_size, dropout):
super().__init__()
self.conv1 = Conv(d_model, d_model, kernel_size=kernel_size, dropout=dropout)
 self.conv2 = Conv(d_model, d_model, kernel_size=kernel_size, dropout=dropout)
 self.relu = nn.ReLU()
self.se = SELayer(d_model)
 def forward(self, src):
h = self.conv2(self.conv1(src))
return self.se(self.relu(src + h))
class FusedEmbedding(nn.Module):
def __init__(self, n_emb):
super().__init__()
self.emb = nn.Embedding(len(TOKEN2INT), n_emb)
 self.n_emb = n_emb
def forward(self, src, se):
embed = self.emb(src)
 embed = embed.reshape(( -1, embed.shape[1], embed.shape[2] * embed.shape[3]))
embed = torch.cat(( embed, se), 2)
 return embed
@property
def d_out(self):
d_emb = 3 * self.n_emb
d_feat = 2 * 5
return d_emb + d_feat
class ConvModel(nn.Module):
def __init__(self, d_emb=50, d_model=256, dropout=0.6, dropout_res=0.4, dropout_emb=0.0,
kernel_size_conv=7, kernel_size_gc=7):
super().__init__()
self.embedding = FusedEmbedding(d_emb)
 self.dropout = nn.Dropout(dropout_emb)
 self.conv = Conv(self.embedding.d_out, d_model, kernel_size=3, dropout=dropout)
 self.block1 = SEResidual(d_model, kernel_size=kernel_size_conv, dropout=dropout_res)
 self.block2 = SEResidual(d_model, kernel_size=kernel_size_conv, dropout=dropout_res)
 self.block3 = SEResidual(d_model, kernel_size=kernel_size_conv, dropout=dropout_res)
 self.block4 = SEResidual(d_model, kernel_size=kernel_size_conv, dropout=dropout_res)
 self.block5 = SEResidual(d_model, kernel_size=kernel_size_conv, dropout=dropout_res)
 self.attn1 = ResidualGraphAttention(d_model, kernel_size=kernel_size_gc, dropout=dropout_res)
 self.attn2 = ResidualGraphAttention(d_model, kernel_size=kernel_size_gc, dropout=dropout_res)
 self.attn3 = ResidualGraphAttention(d_model, kernel_size=kernel_size_gc, dropout=dropout_res)
 self.attn4 = ResidualGraphAttention(d_model, kernel_size=kernel_size_gc, dropout=dropout_res)
 self.linear = nn.Linear(d_model, len(PRED_COLS))
def forward(self,
src: torch.Tensor,
features: torch.Tensor,
bpps: torch.Tensor,
adj: torch.Tensor):
x = self.dropout(self.embedding(src, features))
x = x.permute([0, 2, 1])
 x = self.conv(x)
 x = self.block1(x)
 x = self.attn1(x, adj)
 x = self.block2(x)
 x = self.attn2(x, adj)
 x = self.block3(x)
 x = self.attn3(x, bpps[:, :, :, 0])
 x = self.attn4(x, bpps[:, :, :, 1])
 x = self.block4(x)
 x = self.block5(x)
 x = x.permute([0, 2, 1])
 out = self.linear(x)
 out = torch.clamp(out, -0.5, 1e8)
 return out
class WRMSELoss(nn.Module):
def __init__(self):
super().__init__()
def forward(self, yhat, y, sample_weight=None):
l =(yhat - y)** 2
if sample_weight is not None:
l = l * sample_weight.unsqueeze(dim=1)
 return torch.sqrt(torch.mean(l))
class ColWiseLoss(nn.Module):
def __init__(self, base_loss):
super().__init__()
self.base_loss = base_loss
self.len_scored = 68
def forward(self, yhat, y, column_weight=None, sample_weight=None):
score = 0
for i in range(len(PRED_COLS)) :
s = self.base_loss(
 yhat[:, :self.len_scored, i],
y[:, :self.len_scored, i],
sample_weight 
)/ len(PRED_COLS)
 if column_weight is not None:
s *= column_weight[i]
score += s
return score
class MCRMSELoss(ColWiseLoss):
def __init__(self):
super().__init__(WRMSELoss() )<categorify>
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model
@contextmanager
def timer(name):
s = time.time()
yield
print(f"{name}: {time.time() - s:.3f}sec")
 def pandas_list_to_array(df: pd.DataFrame)-> np.ndarray:
return np.transpose(
 np.array(
 df.values 
.tolist() 
), 
(0, 2, 1)
)
 def preprocess_inputs(df: pd.DataFrame)-> np.ndarray:
return pandas_list_to_array(
 df[['sequence', 'structure', 'predicted_loop_type']] 
.applymap(lambda seq: [TOKEN2INT[x] for x in seq])
)
 def build_adj_matrix(src_df: pd.DataFrame, normalize: bool = True)-> np.ndarray:
n = len(src_df['structure'].iloc[0])
 mat = np.zeros(( len(src_df), n, n))
start_token_indices = []
for r, structure in tqdm(enumerate(src_df['structure'])) :
for i, token in enumerate(structure):
if token == "(":
start_token_indices.append(i)
 elif token == ")":
j = start_token_indices.pop()
mat[r, i, j] = 1
mat[r, j, i] = 1
assert len(start_token_indices)== 0
if normalize:
mat = mat /(mat.sum(axis=2, keepdims=True)+ 1e-8)
 return mat
def replace_data(train_df: pd.DataFrame, test_df: pd.DataFrame, replace_data_dir: str):
print(f"using data from {replace_data_dir}")
 aux = pd.read_csv(replace_data_dir)
 del train_df['structure']
del train_df['predicted_loop_type']
del test_df['structure']
del test_df['predicted_loop_type']
train_df = pd.merge(train_df, aux, on='id', how='left')
 test_df = pd.merge(test_df, aux, on='id', how='left')
 assert len(train_df)== 2400
assert len(test_df)== 3634
assert train_df['structure'].isnull().sum() == 0
assert train_df['predicted_loop_type'].isnull().sum() == 0
assert test_df['structure'].isnull().sum() == 0
assert test_df['predicted_loop_type'].isnull().sum() == 0
return train_df, test_df
def load_bpps(df: pd.DataFrame, data_dir: str)-> np.ndarray:
return np.array([np.load(f'{data_dir}bpps/{did}.npy')for did in df.id])
 def make_bpps_features(bpps_list: List[np.ndarray])-> np.ndarray:
ar = []
for b in bpps_list:
ar.append(b.sum(axis=2))
bpps_sorted = np.sort(b, axis=2)[:, :, ::-1]
ar.append(bpps_sorted[:, :, 0])
 ar.append(bpps_sorted[:, :, 1])
 ar.append(bpps_sorted[:, :, 2])
 bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
nb =(b > 0 ).sum(axis=2)
 nb =(nb - bpps_nb_mean)/ bpps_nb_std
ar.append(nb)
 return np.transpose(np.array(ar),(1, 2, 0))
def make_dataset(device, x: np.ndarray, y: np.ndarray,
bpps_primary: np.ndarray,
bpps_secondary: np.ndarray,
adj_matrix: np.ndarray,
prediction_mask: np.ndarray,
signal_to_noise=None):
x = copy.deepcopy(x)
 if y is not None:
y = copy.deepcopy(y)
 bpps_primary = copy.deepcopy(bpps_primary)
 bpps_secondary = copy.deepcopy(bpps_secondary)
 bpps = np.concatenate([
bpps_primary[:, :, :, np.newaxis],
bpps_secondary[:, :, :, np.newaxis]
], axis=-1)
 adj_matrix = copy.deepcopy(adj_matrix)
 prediction_mask = copy.deepcopy(prediction_mask)
 if y is not None:
y = np.clip(y, -0.5, 10)
 mask = np.abs(y ).max(axis=(1, 2)) < 10
else:
mask = [True] * len(x)
 tensors = [
torch.LongTensor(x[mask]),
torch.Tensor(make_bpps_features([bpps_primary[mask], bpps_secondary[mask]])) ,
torch.Tensor(bpps[mask]),
torch.Tensor(adj_matrix[mask]),
torch.Tensor(prediction_mask[mask])
 ]
if y is not None:
tensors.append(torch.Tensor(y[mask]))
sample_weight = np.clip(np.log(signal_to_noise[mask] + 1.1)/ 2, 0, 100)
 tensors.append(torch.Tensor(sample_weight))
return torch.utils.data.TensorDataset(*[t.to(device)for t in tensors])
 def make_dataset_from_df(device, df: pd.DataFrame, bpps_dir: str, secondary_bpps_dir: str):
assert df['seq_scored'].nunique() == 1
inputs = preprocess_inputs(df)
 bpps = load_bpps(df, bpps_dir)
 adj = build_adj_matrix(df)
 secondary_bpps = load_bpps(df, secondary_bpps_dir)
 mask = np.zeros(( len(df), len(df['sequence'].iloc[0]), len(PRED_COLS)))
 mask[:, :df['seq_scored'].iloc[0], :] = 1
return make_dataset(device, inputs, None, bpps, secondary_bpps, adj, mask)
 def dist(s1: str, s2: str)-> int:
return sum([c1 != c2 for c1, c2 in zip(s1, s2)])
 def get_distance_matrix(s: pd.Series)-> np.ndarray:
mat = np.zeros(( len(s), len(s)))
 for i in tqdm(range(len(s))):
for j in range(i + 1, len(s)) :
mat[i, j] = mat[j, i] = dist(s[i], s[j])
 return mat
def batch_predict(model: nn.Module, loader: DataLoader)-> np.ndarray:
y_preds = np.zeros(( len(loader.dataset), loader.dataset[0][0].shape[0], len(PRED_COLS)))
 for i,(x_batch, x_se, x_bpps, x_adj, y_mask)in enumerate(loader):
y_pred = model(x_batch, x_se, x_bpps, x_adj ).detach() * y_mask
y_preds[i * loader.batch_size:(i + 1)* loader.batch_size, :, :] = y_pred.cpu().numpy()
return y_preds
def calc_loss(y_true: np.ndarray, y_pred: np.ndarray):
err_w_valid = [1 if s in PRED_COLS_SCORED else 0 for s in PRED_COLS]
raw = MCRMSELoss()(torch.Tensor(y_pred), torch.Tensor(y_true), err_w_valid ).item()
return raw * len(PRED_COLS)/ len(PRED_COLS_SCORED )<choose_model_class>
def create_model2(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model
def train_model(model, train_loader, valid_loader, y_valid,
train_epochs, train_loss, verbose=True,
model_path='model'):
params = sum([p.numel() for p in model.parameters() if p.requires_grad])
 print(f'number of params: {params}')
 err_w_train_1 = [1 if s in PRED_COLS_SCORED else 1.0 for s in PRED_COLS]
err_w_train_2 = [1 if s in PRED_COLS_SCORED else 0.01 for s in PRED_COLS]
criterion_train = train_loss
optimizer = torch.optim.Adam(model.parameters())
 losses = []
val_losses = []
y_preds_best = None
for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, x_se, x_bpps, x_adj, y_mask, y_batch, sample_weight in tqdm(train_loader, disable=True):
y_pred = model(x_batch, x_se, x_bpps, x_adj)* y_mask
w = err_w_train_1 if epoch < 30 else err_w_train_2
loss = criterion_train(y_pred, y_batch, w, sample_weight)
 optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
y_preds = batch_predict(model, valid_loader)
 mcloss = calc_loss(y_valid, y_preds)
 val_losses.append(mcloss)
 s = f"{epoch:03d}: trn:{avg_loss:.4f}, clean={mcloss:.4f}, {time.time() - start_time:.2f}s"
losses.append(avg_loss)
 if np.min(val_losses)== mcloss:
y_preds_best = y_preds
torch.save(model.state_dict() , model_path)
 if(isinstance(verbose, bool)and verbose)or(verbose > 0 and(epoch % verbose == 0)) :
print(s)
 print(f'min val_loss: {np.min(val_losses):.4f} at {np.argmin(val_losses)+ 1} epoch')
 model.load_state_dict(torch.load(model_path))
if not verbose:
return np.min(val_losses)
 fig, ax = plt.subplots(1, 3, figsize=(24, 8))
ax[0].plot(np.arange(1, len(losses)+ 1), losses)
 ax[1].plot(np.arange(1, len(val_losses)+ 1), val_losses)
 for i, p in enumerate(PRED_COLS):
ax[2].scatter(y_valid[:, :, i].flatten() , y_preds_best[:, :, i].flatten() , alpha=0.5)
 ax[0].legend(['train'])
 ax[1].legend(['valid(clean)'])
 ax[2].legend(PRED_COLS)
 ax[2].set_xlabel('y_true')
 ax[2].set_ylabel('y_predicted')
 ax[0].set_xlabel('epoch')
 ax[0].set_ylabel('loss')
 ax[1].set_xlabel('epoch')
 ax[1].set_ylabel('loss')
 ax[2].set_xlabel('y_true(clean)')
 ax[2].set_ylabel('y_predicted(clean)')
 plt.show()
return np.min(val_losses )<load_from_csv>
top_feats = range(905)
 print(len(top_feats))
with timer("load data"):
train_df = pd.read_json(DATA_DIR + 'train.json', lines=True)
 test_df = pd.read_json(DATA_DIR + 'test.json', lines=True)
 sample_df = pd.read_csv(DATA_DIR + 'sample_submission.csv')
 train_df, test_df = replace_data(train_df, test_df, REPLACE_DATA_PATH)
 with timer("clustering"):
seq_dist = get_distance_matrix(train_df['sequence'])
 clf = AgglomerativeClustering(n_clusters=None,
distance_threshold=20,
affinity='precomputed',
linkage='average')
 group_index = clf.fit_predict(seq_dist )<categorify>
train=train.astype(float)
 test =test.astype(float )
with timer("preprocess"):
public_df = test_df.query("seq_length != 130")
 private_df = test_df.query("seq_length == 130")
 x = preprocess_inputs(train_df)
 y = pandas_list_to_array(train_df[PRED_COLS])
 label_mask = np.ones_like(y)
 pad = np.zeros(( y.shape[0], x.shape[1] - y.shape[1], y.shape[2]))
y = np.concatenate(( y, pad), axis=1)
 label_mask = np.concatenate(( label_mask, pad), axis=1)
 assert x.shape[1] == y.shape[1]
train_adj = build_adj_matrix(train_df)
 primary_bpps = load_bpps(train_df, PRIMARY_BPPS_DIR)
 secondary_bpps = load_bpps(train_df, SECONDARY_BPPS_DIR)
 public_data = make_dataset_from_df(device, public_df, PRIMARY_BPPS_DIR, SECONDARY_BPPS_DIR)
 private_data = make_dataset_from_df(device, private_df, PRIMARY_BPPS_DIR, SECONDARY_BPPS_DIR )<load_pretrained>
N_STARTS = 2
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=4, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 if seed%2 == 0:
model = create_model1(len(top_feats))
else :
model = create_model2(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=35, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS
kf = StratifiedGroupKFold(NFOLDS, random_state=42, shuffle=True)
 pred_oof = np.zeros_like(y)
 pred_public = np.zeros(( len(public_data), len(public_df['sequence'].iloc[0]), len(PRED_COLS)))
 pred_private = np.zeros(( len(private_data), len(private_df['sequence'].iloc[0]), len(PRED_COLS)))
 public_loader = DataLoader(public_data, batch_size=128, shuffle=False)
 private_loader = DataLoader(private_data, batch_size=128, shuffle=False)
 clean_idx = [i for i in range(len(train_df)) if train_df['SN_filter'].iloc[i]]
sn_mask = train_df['SN_filter'] == 1
criterion_train = MCRMSELoss()
model_path = "model_fold{}"
losses = []
for i,(train_index, valid_index)in enumerate(kf.split(x, train_df['SN_filter'], groups=group_index)) :
print(f'fold {i}')
 model = ConvModel().to(device)
 s = time.time()
train_data = make_dataset(device, x[train_index], y[train_index],
primary_bpps[train_index], secondary_bpps[train_index],
train_adj[train_index],
label_mask[train_index],
signal_to_noise=train_df['signal_to_noise'][train_index].values)
 valid_index_c = [v for v in valid_index if v in clean_idx]
valid_data_clean = make_dataset(device, x[valid_index_c], None,
primary_bpps[valid_index_c],
secondary_bpps[valid_index_c],
train_adj[valid_index_c],
label_mask[valid_index_c])
 valid_data_noisy = make_dataset(device, x[valid_index], None,
primary_bpps[valid_index],
secondary_bpps[valid_index],
train_adj[valid_index],
label_mask[valid_index])
 train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)
 valid_loader_clean = DataLoader(valid_data_clean, batch_size=128, shuffle=False)
 valid_loader_noisy = DataLoader(valid_data_noisy, batch_size=128, shuffle=False)
 loss = train_model(model,
train_loader,
valid_loader_clean,
y[valid_index_c],
TRAIN_EPOCHS,
criterion_train,
verbose=5,
model_path=model_path.format(i))
losses.append(loss)
 pred_oof[valid_index] = batch_predict(model, valid_loader_noisy)
 pred_public += batch_predict(model, public_loader)/ NFOLDS
pred_private += batch_predict(model, private_loader)/ NFOLDS
print(f'elapsed: {time.time() - s:.1f}sec')
 oof_score = calc_loss(y, pred_oof)
 print(f'oof(all): {oof_score:.4f}')
 oof_score = calc_loss(y[sn_mask], pred_oof[sn_mask])
 print(f'oof(clean): {oof_score:.4f}')
 preds_ls = []
for df, preds in [(public_df, pred_public),(private_df, pred_private)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=PRED_COLS)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.head()
submission = sample_df[['id_seqpos']].merge(preds_df, on=['id_seqpos'])
 submission.to_csv('submission.csv', index=False)
 np.save('oof', pred_oof)
 np.save('public', pred_public)
 np.save('private', pred_private)
 print(losses )<load_from_csv>
print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )
path = '.. /input/stanford-covid-vaccine/'
bbps_folder = f'{path}bpps/'
train = pd.read_json(f'{path}train.json', lines=True)
 test = pd.read_json(f'{path}test.json', lines=True)
 sample_sub = pd.read_csv(f'{path}sample_submission.csv' )<data_type_conversions>
old_list = []
avg_list = []
new_list = []
for tar in range(206):
score1 = log_loss(train_targets.loc[:, train_targets.columns[tar]], valid_results.loc[:, train_targets.columns[tar]])
 score2 = log_loss(train_targets.loc[:, train_targets.columns[tar]], res_NN.loc[:, train_targets.columns[tar]])
 if score2 >= score1:
ss_NN.loc[:, train_targets.columns[tar]] = sub.loc[:, train_targets.columns[tar]]
res_NN.loc[:, train_targets.columns[tar]] = valid_results.loc[:, train_targets.columns[tar]]
else :
old_list.append(tar)
targets = np.stack([train[col].to_list() for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C','deg_pH10', 'deg_50C']], axis=-1)
 filter_1 = pd.Series(( targets > -0.5 ).all(axis=(1,2)) ).astype(int)
 filter_2 =(train['signal_to_noise'] > 1 ).astype(int)
 reactivity_bins = pd.qcut(train['reactivity'].map(np.mean), 5, labels=False)
 deg_Mg_pH10_bins = pd.qcut(train['deg_Mg_pH10'].map(np.mean), 5, labels=False)
 deg_Mg_50C_bins = pd.qcut(train['deg_Mg_50C'].map(np.mean), 5, labels=False)
 stratify = pd.concat([filter_1, filter_2, reactivity_bins, deg_Mg_50C_bins, deg_Mg_pH10_bins], axis=1)
 stratify = stratify.astype(str ).apply(''.join, axis=1)
 stratify.value_counts()<split>
score1 = 1.- log_loss_metric(train_targets, res_NN)
 score2 = 1.- 0.014921110834529303
scores = [score1,score2]
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]
skf= StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 folds = list(skf.split(train, stratify))<prepare_x_and_y>
ss_NN.loc[:, train_targets.columns[:]] = ss_NN.loc[:, train_targets.columns[:]] * weights[0] + sub.loc[:, train_targets.columns[:]] * weights[1]
res_NN.loc[:, train_targets.columns[:]] = res_NN.loc[:, train_targets.columns[:]] * weights[0] + valid_results.loc[:, train_targets.columns[:]] * weights[1]
def get_structure_matrix(structure):
n = len(structure)
 structure_matrix = np.zeros(( n,n), int)
 for i in range(n):
if structure[i] == '(':
count = 0
for j in range(i, n):
if structure[j] == '(':
count += 1
elif structure[j] == ')':
count -= 1
if count==0:
structure_matrix[i,j]=1
structure_matrix[j,i]=1
break
return structure_matrix.astype(int)
 def get_stracture_bpps(structure_matrix, bbps):
stracture_bbps = np.zeros(structure_matrix.shape[:-1])
 stracture_bbps[structure_matrix.any(axis=0)] = bbps[structure_matrix.astype(bool)]
return stracture_bbps
def get_distance2pair(structure_matrix):
self_idx = np.arange(structure_matrix.shape[0])
 idx_pair = structure_matrix.argmax(axis=1)
 idx_pair[~structure_matrix.any(axis=1)] = self_idx[~structure_matrix.any(axis=1)]
distance2pair =(idx_pair - self_idx)/(structure_matrix.shape[0]-1)
 return distance2pair
def proc_df(df):
df = deepcopy(df)
 df['bbps'] = df['id'].apply(lambda x: np.load(f'{bbps_folder}{x}.npy'))
df['bbps_sum'] = df['bbps'].apply(lambda x: x.sum(axis=1))
df['bbps_max'] = df['bbps'].apply(lambda x: x.max(axis=1))
df['structure_matrix'] = df['structure'].apply(get_structure_matrix)
 df['structure_bpps'] = df.apply(lambda x: get_stracture_bpps(x['structure_matrix'], x['bbps']), axis=1)
 df['distance2pair'] = df['structure_matrix'].apply(get_distance2pair)
 return df
STRUCTURE_CODE = {'(': 0, '.': 1, ')': 2}
PREDICTED_LOOP_TYPE_CODE = {'H': 0, 'E': 1, 'B': 2, 'M': 3, 'X': 4, 'S': 5, 'I': 6}
SEQUANCE_CODE = {'U': 0, 'C': 1, 'A': 2, 'G': 3}
ERROR_COLUMNS = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_Mg_50C', 'deg_error_pH10', 'deg_error_50C']
TARGET_COLUMNS = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
class CompDataset(Dataset):
def __init__(self, data, bbps_folder, device='cpu', train_mode=True):
data = proc_df(data)
 self.data = {}
self.data['sequence'] = LongTensor(np.array([[SEQUANCE_CODE[t] for t in s] for s in data['sequence']]))
self.data['structure'] = LongTensor(np.array([[STRUCTURE_CODE[t] for t in s] for s in data['structure']]))
self.data['predicted_loop_type'] = LongTensor(np.array([[PREDICTED_LOOP_TYPE_CODE[t] for t in lt] for lt in data['predicted_loop_type']]))
self.data['bbps'] = FloatTensor(np.array(data['bbps'].to_list()))
self.data['structure_matrix'] = LongTensor(np.array(data['structure_matrix'].to_list()))
bbps_sum = FloatTensor(np.array(data['bbps_sum'].to_list()))
bbps_max = FloatTensor(np.array(data['bbps_max'].to_list()))
structure_bpps = FloatTensor(np.array(data['structure_bpps'].to_list()))
distance2pair = FloatTensor(np.array(data['distance2pair'].to_list()))
self.data['features'] = torch.stack([bbps_sum, bbps_max, structure_bpps, distance2pair], dim=1)
 if train_mode:
error = np.stack([data[col].tolist() for col in ERROR_COLUMNS], axis=1 ).swapaxes(1,2)
 target = np.stack([data[col].tolist() for col in TARGET_COLUMNS], axis=1 ).swapaxes(1,2)
 self.data['targets'] = torch.cat([FloatTensor(target), torch.zeros(( target.shape[0],39,5)) ], dim=1)
 self.data['error'] = torch.cat([FloatTensor(error), torch.ones(( target.shape[0],39,5)) *5], dim=1)
 self.data['signal_to_noise'] = FloatTensor(data['signal_to_noise'].to_numpy())
 self.data['sn_filter'] = LongTensor(data['SN_filter'].to_numpy())
 self.data['filter_1'] = LongTensor(( target.min(-1)> -0.5 ).all(-1))
self.data['filter_2'] = LongTensor(target.mean(( 1,2)) /error.mean(( 1,2)) > 1)
 for key in self.data.keys() :
self.data[key] = self.data[key].to(device)
 def __len__(self):
return self.data['sequence'].shape[0]
def __getitem__(self, idx: int):
return {k: v[idx] for k,v in self.data.items() }<feature_engineering>
print(f'pytorch OOF Metric with postprocessing: {log_loss_metric(train_targets, valid_results)}' )
WSMin = 48
def random_window_collate_fn(batch):
seq_len = batch[0]['sequence'].shape[0]
windowed_batch = []
window_size = np.random.randint(WSMin, seq_len)
 for el in batch:
window_left = np.random.randint(seq_len-window_size)
 window_right = window_left+window_size
el['features'] = el['features'][:, window_left:window_right]
for k in ['sequence', 'structure', 'predicted_loop_type', 'targets', 'error']:
if k in el.keys() :
el[k] = el[k][window_left:window_right]
for k in ['bbps', 'structure_matrix']:
el[k] = el[k][window_left:window_right, window_left:window_right]
windowed_batch.append(el)
 new_batch = {}
for k in batch[0].keys() :
new_batch[k] = torch.stack([el[k] for el in windowed_batch])
 return new_batch<compute_test_metric>
print(f'Final OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )
MEDIAN_ERROR = 0.105
class CompLoss(_Loss):
def forward(self, logits, targets, error, filter_1, filter_2):
logits, targets, error = logits[:,:,:3], targets[:,:,:3], error[:,:,:3]
weights = MEDIAN_ERROR/(MEDIAN_ERROR + error)
 loss =(logits - targets)**2
loss = loss*weights
loss = loss.mean(1)
 loss = torch.sqrt(loss)
 loss = loss.mean(0)
 loss = loss.mean(0)
 return loss
def comp_metric_fn(logits, targets, error, filter_1, filter_2):
logits, targets, error = logits[:,:,:3], targets[:,:,:3], error[:,:,:3]
logits, targets, error = logits[(filter_1==1)&(filter_2==1)], targets[(filter_1==1)&(filter_2==1)], error[(filter_1==1)&(filter_2==1)]
weights =(error < 5 ).float()
loss =(logits - targets)**2
loss = loss.sum(1)/weights.sum(1)
 loss = loss.mean(0)
 loss = torch.sqrt(loss)
 loss = loss.mean(0)
 return loss.item()
class PreTrainLoss(_Loss):
def forward(self, logits, sequence):
logits = logits.reshape(-1,4)
 sequence = sequence.reshape(-1)
 return F.cross_entropy(logits, sequence )<prepare_x_and_y>
ss_NN.to_csv('submission.csv', index=False )