Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<define_variables>
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()
BASE_PATH = "/kaggle/input/stanford-covid-vaccine"
MODEL_SAVE_PATH = "/kaggle/model"
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 print(Ds.shape)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss
def create_loader(df, batch_size=1, is_test=False):
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, drop_last=False)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False, drop_last=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index])
<choose_model_class>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True)
 def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test)
 kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class Conv2dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv2dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv2d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, 128, 3, padding=1)
 self.conv1 = Conv1dStack(128, 64, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(64, 32, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(32, 32, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class BppAttn(nn.Module):
def __init__(self, in_channel: int, out_channel: int):
super(BppAttn, self ).__init__()
self.conv0 = Conv1dStack(in_channel, out_channel, 3, padding=1)
 self.bpp_conv = Conv2dStack(5, out_channel)
 def forward(self, x, bpp):
x = self.conv0(x)
 bpp = self.bpp_conv(bpp)
 x = torch.matmul(bpp, x.unsqueeze(-1))
return x.squeeze(-1)
 class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x)
 class TransformerWrapper(nn.Module):
def __init__(self, dmodel=256, nhead=8, num_layers=2):
super(TransformerWrapper, self ).__init__()
self.pos_encoder = PositionalEncoding(256)
 encoder_layer = TransformerEncoderLayer(d_model=dmodel, nhead=nhead)
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.pos_emb = PositionalEncoding(dmodel)
 def flatten_parameters(self):
pass
def forward(self, x):
x = x.permute(( 1, 0, 2)).contiguous()
x = self.pos_emb(x)
 x = self.transformer_encoder(x)
 x = x.permute(( 1, 0, 2)).contiguous()
return x, None
class RnnLayers(nn.Module):
def __init__(self, dmodel, dropout=0.3, transformer_layers: int = 2):
super(RnnLayers, self ).__init__()
self.dropout = nn.Dropout(dropout)
 self.rnn0 = TransformerWrapper(dmodel, nhead=8, num_layers=transformer_layers)
 self.rnn1 = nn.LSTM(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 self.rnn2 = nn.GRU(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 def forward(self, x):
self.rnn0.flatten_parameters()
x, _ = self.rnn0(x)
 if self.rnn1 is not None:
self.rnn1.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn1(x)
 if self.rnn2 is not None:
self.rnn2.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn2(x)
 return x
class BaseAttnModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(BaseAttnModel, self ).__init__()
self.linear0 = nn.Linear(14 + 3, 1)
 self.seq_encoder_x = SeqEncoder(18)
 self.attn = BppAttn(256, 128)
 self.seq_encoder_bpp = SeqEncoder(128)
 self.seq = RnnLayers(256 * 2, dropout=0.3,
transformer_layers=transformer_layers)
 def forward(self, x, bpp):
bpp_features = get_bpp_feature(bpp[:, :, :, 0].float())
 x = torch.cat([x] + bpp_features, dim=-1)
 learned = self.linear0(x)
 x = torch.cat([x, learned], dim=-1)
 x = x.permute(0, 2, 1 ).contiguous().float()
bpp = bpp.permute([0, 3, 1, 2] ).contiguous().float()
x = self.seq_encoder_x(x)
 bpp = self.attn(x, bpp)
 bpp = self.seq_encoder_bpp(bpp)
 x = x.permute(0, 2, 1 ).contiguous()
bpp = bpp.permute(0, 2, 1 ).contiguous()
x = torch.cat([x, bpp], dim=2)
 x = self.seq(x)
 return x
class AEModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(AEModel, self ).__init__()
self.seq = BaseAttnModel(transformer_layers=transformer_layers)
 self.linear = nn.Sequential(
 nn.Linear(256 * 2, 14),
nn.Sigmoid() , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = F.dropout(x, p=0.3)
 x = self.linear(x)
 return x
class FromAeModel(nn.Module):
def __init__(self, seq, pred_len=68, dmodel: int = 256):
super(FromAeModel, self ).__init__()
self.seq = seq
self.pred_len = pred_len
self.linear = nn.Sequential(
 nn.Linear(dmodel * 2, len(target_cols)) , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = self.linear(x)
 x = x[:, :self.pred_len]
return x 
<load_from_disk>
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100, hidden_dim=256, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 return model
keras_model = build_model()
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv0.h5')
 keras_model.layers
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_train_data.head()
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
features, _ = preprocess(base_train_data, True)
 features_tensor = torch.from_numpy(features)
 dataset0 = VacDataset(features_tensor, base_train_data, None)
 features, _ = preprocess(public_df, True)
 features_tensor = torch.from_numpy(features)
 dataset1 = VacDataset(features_tensor, public_df, None)
 features, _ = preprocess(private_df, True)
 features_tensor = torch.from_numpy(features)
 dataset2 = VacDataset(features_tensor, private_df, None)
 loader0 = torch.utils.data.DataLoader(dataset0, BATCH_SIZE, shuffle=False, drop_last=False)
 loader1 = torch.utils.data.DataLoader(dataset1, BATCH_SIZE, shuffle=False, drop_last=False)
 loader2 = torch.utils.data.DataLoader(dataset2, BATCH_SIZE, shuffle=False, drop_last=False )<find_best_model_class>
device = torch.device('cuda:%s'%0 if torch.cuda.is_available() else 'cpu')
 def Init_params(shape,w=None,b=None):
if w is None:
w = torch.nn.Parameter(torch.empty(*shape))
nn.init.xavier_uniform_(w)
 else:
w = torch.nn.Parameter(w)
 if b is None:
b = torch.nn.Parameter(torch.zeros(shape[1]))
else:
b = torch.nn.Parameter(b)
 return w,b
class GRU(nn.Module):
def __init__(self,input_dim,hidden_dim,w_i=None,b_i=None,w_h=None,b_h=None):
super(GRU, self ).__init__()
self.w_i,self.b_i = Init_params([input_dim,3*hidden_dim],w_i,b_i)
 self.w_h,self.b_h = Init_params([hidden_dim,3*hidden_dim],w_h,b_h)
 self.hd = hidden_dim
def forward(self,x):
hidden = torch.zeros(( x.shape[0], self.hd)).to(device)
 output = []
for i in range(x.shape[1]):
x_z = torch.matmul(x[:,i,:],self.w_i[:,:self.hd])+ self.b_i[:self.hd]
x_r = torch.matmul(x[:,i,:],self.w_i[:,self.hd:2*self.hd])+ self.b_i[self.hd:2*self.hd]
x_n = torch.matmul(x[:,i,:],self.w_i[:,2*self.hd:])+ self.b_i[2*self.hd:]
h_z = torch.matmul(hidden,self.w_h[:,:self.hd])+ self.b_h[:self.hd]
h_r = torch.matmul(hidden,self.w_h[:,self.hd:2*self.hd])+ self.b_h[self.hd:2*self.hd]
h_n = torch.matmul(hidden,self.w_h[:,2*self.hd:])+ self.b_h[2*self.hd:]
z = torch.sigmoid(x_z+h_z)
 r = torch.sigmoid(x_r+h_r)
 n = torch.tanh(x_n+r*h_n)
 h =(1-z)*n + z*hidden
hidden = h
output.append(h.unsqueeze(1))
return torch.cat(output,1)
 class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
num_target=5
w = torch.Tensor(keras_model.layers[2].get_weights() [0])
 self.cate_emb = nn.Embedding.from_pretrained(w,freeze=False)
 self.gru = GRU(100*3+3, 256, torch.Tensor(keras_model.layers[-4].get_weights() [0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [2][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [1] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [2][1] ).contiguous())
 self.reverse_gru = GRU(100*3+3, 256, torch.Tensor(keras_model.layers[-4].get_weights() [3] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [5][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [4] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [5][1] ).contiguous())
 self.gru1 = GRU(512, 256, torch.Tensor(keras_model.layers[-3].get_weights() [0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [2][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [1] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [2][1] ).contiguous())
 self.reverse_gru1 = GRU(512, 256, torch.Tensor(keras_model.layers[-3].get_weights() [3] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [5][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [4] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [5][1] ).contiguous())
 self.predict = nn.Linear(512,num_target)
 for i,(n,p)in enumerate(self.predict.named_parameters()):
if i == 0:
p.data = torch.nn.Parameter(torch.Tensor(keras_model.layers[-1].get_weights() [0].T ).contiguous())
 if i == 1:
p.data = torch.nn.Parameter(torch.Tensor(keras_model.layers[-1].get_weights() [1] ).contiguous())
 def forward(self, cateX,contX):
cate_x = self.cate_emb(cateX ).view(cateX.shape[0],cateX.shape[1],-1)
 sequence = torch.cat([cate_x,contX],-1)
 x = self.gru(sequence)
 reverse_x = torch.flip(self.reverse_gru(torch.flip(sequence,[1])) ,[1])
 sequence = torch.cat([x,reverse_x],-1)
 x = self.gru1(sequence)
 reverse_x = torch.flip(self.reverse_gru1(torch.flip(sequence,[1])) ,[1])
 x = torch.cat([x,reverse_x],-1)
 x = F.dropout(x,0.5,training=self.training)
 predict = self.predict(x)
 return predict
pytorch_model = Net()
pytorch_model.to(device )
def learn_from_batch_ae(model, data, device):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(device), data["bpp"].to(device))
loss = F.binary_cross_entropy(out, target.to(device))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, start_it: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data, device)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'epoch: {epoch} loss: {loss_m}')
 losses = []
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
<choose_model_class>
x = preprocess_inputs(train[:1])
 cate_x = torch.LongTensor(x[:,:,:3] ).to(device)
 cont_x = torch.Tensor(x[:,:,3:] ).to(device)
 y = np.array(train[:1][pred_cols].values.tolist() ).transpose(( 0, 2, 1))
set_seed(123)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 lr_scheduler = None
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AEModel()
model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
res = train_ae(model, loader0, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 epoch = res["min_loss_epoch"]
shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"model-{epoch}.pt"), "ae-model.pt" )<compute_test_metric>
keras_y = keras_model.predict(x)
 keras_y
def MCRMSE(y_true, y_pred):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse), dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise):
loss = MCRMSE(target, predict)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 loss =(loss * weight ).mean()
return loss
def learn_from_batch(model, data, optimizer, lr_scheduler, device):
optimizer.zero_grad()
out = model(data["sequence"].to(device), data["bpp"].to(device))
signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(out, data["label"].to(device), signal_to_noise.to(device))
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, device):
model.eval()
loss_list = []
mcrmse = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(data["sequence"].to(device), data["bpp"].to(device))
mcrmse_ = MCRMSE(data["label"].to(device), y)[data["signal_to_noise"] > 1]
mcrmse.append(mcrmse_.mean().item())
 loss = sn_mcrmse_loss(y, data["label"].to(device), data["signal_to_noise"].to(device))
loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcmse=np.mean(mcrmse))
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
writer = SummaryWriter(log_path)
 it = 0
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 min_eval_loss = 10.0
min_eval_epoch = None
for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, device)
 loss_v = loss.item()
writer.add_scalar('loss', loss_v, it)
 losses.append(loss_v)
 it += 1
print(f'epoch: {epoch} loss: {np.mean(losses)}')
 losses = []
eval_result = evaluate(model, valid_data, device)
 eval_loss = eval_result["loss"]
if eval_loss <= min_eval_loss:
min_eval_epoch = epoch
min_eval_loss = eval_loss
print(f"eval loss: {eval_loss} {eval_result['mcmse']}")
 writer.add_scalar(f"evaluate/loss", eval_loss, epoch)
 writer.add_scalar(f"evaluate/mcmse", eval_result["mcmse"], epoch)
 model.train()
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
print(f'min eval loss: {min_eval_loss} epoch {min_eval_epoch}')
 return min_eval_epoch 
<load_pretrained>
pytorch_model.eval()
pytorch_y = pytorch_model(cate_x,cont_x ).detach().cpu().numpy()
pytorch_y
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 samples = base_train_data
save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 split = ShuffleSplit(n_splits=5, test_size=.1)
 ids = samples.reset_index() ["id"]
set_seed(124)
 for fold,(train_index, test_index)in enumerate(split.split(samples)) :
print(f"fold: {fold}")
 train_df = samples.loc[train_index].reset_index()
val_df = samples.loc[test_index].reset_index()
train_loader = create_loader(train_df, BATCH_SIZE)
 valid_loader = create_loader(val_df, BATCH_SIZE)
 print(train_df.shape, val_df.shape)
 ae_model = AEModel()
state_dict = torch.load("./ae-model.pt")
 ae_model.load_state_dict(state_dict)
 del state_dict
model = FromAeModel(ae_model.seq)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
epoch = train(model, train_loader, valid_loader, optimizer, lr_scheduler, 200, device=device,
log_path=f"logs/{fold}")
 shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"./model-{epoch}.pt"), f"model_prediction/model-{fold}.pt")
 del model<predict_on_test>
np.mean(np.abs(keras_y-pytorch_y[:,:68,:]))
def predict_batch(model, data, device):
with torch.no_grad() :
pred = model(data["sequence"].to(device), data["bpp"].to(device))
pred = pred.detach().cpu().numpy()
return_values = []
ids = data["ids"]
for idx, p in enumerate(pred):
id_ = ids[idx]
assert p.shape ==(model.pred_len, len(target_cols))
for seqpos, val in enumerate(p):
assert len(val)== len(target_cols)
 dic = {key: val for key, val in zip(target_cols, val)}
dic["id_seqpos"] = f"{id_}_{seqpos}"
return_values.append(dic)
 return return_values
def predict_data(model, loader, device, batch_size):
data_list = []
for i, data in enumerate(progress_bar(loader)) :
data_list += predict_batch(model, data, device)
 expected_length = model.pred_len * len(loader)* batch_size
assert len(data_list)== expected_length, f"len = {len(data_list)} expected = {expected_length}"
return data_list 
<load_pretrained>
gkf = GroupKFold(n_splits=5)
 keras_predict = []
pytorch_predict = []
targets = []
for fold,(train_index, valid_index)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv%s.h5'%fold)
 t_valid = train.iloc[valid_index]
t_valid = t_valid[t_valid['SN_filter'] == 1]
valid_x = preprocess_inputs(t_valid)
 valid_count = valid_x.shape[0]
valid_cate_x = torch.LongTensor(valid_x[:,:,:3])
 valid_cont_x = torch.Tensor(valid_x[:,:,3:])
 valid_y = torch.Tensor(np.array(t_valid[pred_cols].values.tolist() ).transpose(( 0, 2, 1)))
 valid_data = TensorDataset(valid_cate_x,valid_cont_x,valid_y)
 valid_data_loader = DataLoader(dataset=valid_data,shuffle=False,batch_size=32,num_workers=1)
 valid_y = valid_y.numpy()
targets.append(valid_y)
 keras_predict.append(keras_model.predict(valid_x))
pytorch_model = Net()
pytorch_model.to(device)
 pytorch_model.eval()
all_pred = []
for data in valid_data_loader:
cate_x,cont_x,y = [x.to(device)for x in data]
outputs = pytorch_model(cate_x,cont_x)
 all_pred.append(outputs.detach().cpu().numpy())
 all_pred = np.concatenate(all_pred,0)[:,:68,:]
pytorch_predict.append(all_pred )
device = torch.device('cuda')if torch.cuda.is_available() else "cpu"
BATCH_SIZE = 1
base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 pred_df_list = []
c = 0
for fold in range(5):
model_load_path = f"./model_prediction/model-{fold}.pt"
ae_model0 = AEModel()
ae_model1 = AEModel()
model_pub = FromAeModel(pred_len=107, seq=ae_model0.seq)
 model_pub = model_pub.to(device)
 model_pri = FromAeModel(pred_len=130, seq=ae_model1.seq)
 model_pri = model_pri.to(device)
 state_dict = torch.load(model_load_path, map_location=device)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
data_list = []
data_list += predict_data(model_pub, pub_loader, device, BATCH_SIZE)
 data_list += predict_data(model_pri, pri_loader, device, BATCH_SIZE)
 pred_df = pd.DataFrame(data_list, columns=["id_seqpos"] + target_cols)
 print(pred_df.head())
 print(pred_df.tail())
 pred_df_list.append(pred_df)
 c += 1
data_dic = dict(id_seqpos=pred_df_list[0]["id_seqpos"])
 for col in target_cols:
vals = np.zeros(pred_df_list[0][col].shape[0])
 for df in pred_df_list:
vals += df[col].values
data_dic[col] = vals / float(c)
 pred_df_avg = pd.DataFrame(data_dic, columns=["id_seqpos"] + target_cols)
 print(pred_df_avg.head())
 pred_df_avg.to_csv("./submission.csv", index=False)
<set_options>
for i in range(5):
print('fold %s output difference between Keras and Pytorch:'%i,np.mean(np.abs(keras_predict[i]-pytorch_predict[i])) )
warnings.filterwarnings("ignore" )<define_variables>
def Metric(target,pred):
metric = 0
for i in range(target.shape[-1]):
metric +=(np.sqrt(np.mean(( target[:,:,i]-pred[:,:,i])**2)) /target.shape[-1])
 return metric
seed = 42<train_on_grid>
for i in range(5):
print('fold %s'%i,'|','metric of keras outputs:%.6f'%Metric(targets[i],keras_predict[i]),'|','metric of pytorch outputs:%.6f'%Metric(targets[i],pytorch_predict[i]))
DEVICE = "TPU"
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print('Number of devices: {}'.format(strategy.num_replicas_in_sync))<define_variables>
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=5):
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
for i in range(5):
print('fold %s'%i,mcrmse(targets[i],keras_predict[i]))
dropout_model = 0.36
hidden_dim_first = 128
hidden_dim_second = 256
hidden_dim_third = 128<load_from_csv>
for i in range(5):
print('fold %s'%i,K.mean(mcrmse(targets[i],keras_predict[i])) )
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 print("train shape",train.shape)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
<load_pretrained>
def Pred(df):
test_x = preprocess_inputs(df)
 test_cate_x = torch.LongTensor(test_x[:,:,:3])
 test_cont_x = torch.Tensor(test_x[:,:,3:])
 test_data = TensorDataset(test_cate_x,test_cont_x)
 test_data_loader = DataLoader(dataset=test_data,shuffle=False,batch_size=64,num_workers=1)
 all_id = []
for i,row in df.iterrows() :
for j in range(row['seq_length']):
all_id.append(row['id']+'_%s'%j)
 all_id = np.array(all_id ).reshape(-1,1)
 all_pred = np.zeros(len(all_id)*5 ).reshape(len(all_id),5)
 for fold in range(5):
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv%s.h5'%fold)
 model = Net()
model.to(device)
 model.eval()
t_all_pred = []
for data in test_data_loader:
cate_x,cont_x = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 t_all_pred.append(outputs.detach().cpu().numpy())
 t_all_pred = np.concatenate(t_all_pred,0)
 all_pred += t_all_pred.reshape(-1,5)
 all_pred /= 5
sub = pd.DataFrame(all_pred,columns=['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C'])
 sub['id_seqpos'] = all_id
return sub
public_sub = Pred(test.loc[test['seq_length']==107])
 private_sub = Pred(test.loc[test['seq_length']==130])
 pytorch_sub = pd.concat([public_sub,private_sub] ).reset_index(drop=True)
 pytorch_sub = pytorch_sub[['id_seqpos']+pred_cols]
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr 
<feature_engineering>
pytorch_sub = pytorch_sub.sort_values(by=['id_seqpos'] ).reset_index(drop=True )
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )<define_variables>
keras_sub = pd.read_csv('.. /input/gru-lstm-with-feature-engineering-and-augmentation/submission.csv' )
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols=['sequence','predicted_loop_type','structure']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
_, position_fea = np.mgrid[0:bpps_nb_fea.shape[0]:1, 0:bpps_nb_fea.shape[1]:1]/(bpps_nb_fea.shape[1]-1)
 return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea, position_fea[:,:,np.newaxis]], 2)
 kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_<load_from_csv>
keras_sub = keras_sub.sort_values(by=['id_seqpos'] ).reset_index(drop=True )
aug_df = pd.read_csv('.. /input/augmented-data-for-stanford-covid-vaccine/48k_augment.csv')
 display(aug_df.head() )<merge>
np.mean(np.abs(keras_sub[pred_cols].values-pytorch_sub[pred_cols].values))
<define_variables><EOS> 
pytorch_sub.to_csv('./submission.csv',index=False )