Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
print("Last updated:")
 print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
aug_df = pd.read_csv('.. /input/covid-data/aug_data.csv' )<merge>
init_notebook_mode(connected=True)
def augment_data(df, concat=True):
df = df.copy()
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left' ).sort_values('index')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
if concat:
df = df.append(new_df[df.columns] ).reset_index(drop=True)
 return df
else:
return new_df[df.columns].reset_index(drop=True )<define_variables>
train_df=pd.read_csv(PATH+'train.csv')
 test_df=pd.read_csv(PATH+'test.csv' )
PUBLIC_IDS = public_df['id'].values
def is_public(id_seqpos):
id_ = '_'.join(id_seqpos.split('_')[:2])
 return id_ in PUBLIC_IDS<define_variables>
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))
print("Test: rows:{} cols:{}".format(test_df.shape[0], test_df.shape[1]))
PL_PATH = ".. /input/covid-pl/"<load_pretrained>
def missing_data(data):
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() /data.isnull().count() *100 ).sort_values(ascending = False)
 return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data(train_df )
PL_PUBLIC = np.load(PL_PATH + 'pl_public.npy')
 PL_PRIVATE = np.load(PL_PATH + 'pl_private.npy' )<categorify>
missing_data(test_df )
for t, target in enumerate(TARGETS):
tgt = []
for i in range(len(public_df)) :
tgt.append(list(PL_PUBLIC[i, :SEQ_SCORED_PUBLIC, t]))
public_df[target] = tgt
tgt = []
for i in range(len(private_df)) :
tgt.append(list(PL_PRIVATE[i, :SEQ_SCORED_PRIVATE, t]))
private_df[target] = tgt
public_df['signal_to_noise'] = 1
private_df['signal_to_noise'] = 1
public_df = augment_data(public_df)
 private_df = augment_data(private_df )<load_pretrained>
def get_categories(data, val):
tmp = data[val].value_counts()
return pd.DataFrame(data={'Number': tmp.values}, index=tmp.index ).reset_index()
def save_model_weights(model, filename, verbose=1, cp_folder=""):

 if verbose:
print(f"
-> Saving weights to {os.path.join(cp_folder, filename)}
")
 torch.save(model.state_dict() , os.path.join(cp_folder, filename))
def load_model_weights(model, filename, verbose=1, cp_folder=""):

 if verbose:
print(f"
-> Loading weights from {os.path.join(cp_folder,filename)}
")
 try:
model.load_state_dict(os.path.join(cp_folder, filename), strict=strict)
 except BaseException:
model.load_state_dict(
 torch.load(os.path.join(cp_folder, filename), map_location="cpu"),
strict=True, 
)
 return model<categorify>
def get_survived_categories(data, val):
tmp = data.groupby('Survived')[val].value_counts()
return pd.DataFrame(data={'Number': tmp.values}, index=tmp.index ).reset_index()
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss<create_dataframe>
train_df['Ticket'].value_counts().head(10 )
def create_loader(df, batch_size=64, is_test=False, shuffle=True):
if is_test:
shuffle = False
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=shuffle)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index] )<init_hyperparams>
train_df['Cabin'].value_counts().head(10 )
USE_FT = True
CNN_DROP = 0.1
ENC_DROP = 0.1
RNN_DROP = 0.3
LOGIT_DROP = 0.25
D = 256<concatenate>
tmp = train_df.groupby(['SibSp', 'Parch'])['Survived'].value_counts()
df = pd.DataFrame(data={'Passengers': tmp.values}, index=tmp.index ).reset_index()
hover_text = []
for index, row in df.iterrows() :
hover_text.append(( 'Sibilings: {}
'+
'Parents/Children: {}
'+
'Survived: {}
'+
'Passengers: {}' ).format(row['SibSp'],
row['Parch'],
row['Survived'],
row['Passengers']))
df['hover_text'] = hover_text
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class Conv2dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv2dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv2d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(CNN_DROP),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int, out_dim=256):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, out_dim // 2, 3, padding=1)
 self.conv1 = Conv1dStack(out_dim // 2, out_dim // 4, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(out_dim // 4, out_dim // 8, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(out_dim // 8, out_dim // 8, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class BppAttn(nn.Module):
def __init__(self, in_channel: int, out_channel: int):
super(BppAttn, self ).__init__()
self.conv0 = Conv1dStack(in_channel, out_channel, 3, padding=1)
 self.bpp_conv = Conv2dStack(5, out_channel)
 def forward(self, x, bpp):
x = self.conv0(x)
 bpp = self.bpp_conv(bpp)
 x = torch.matmul(bpp, x.unsqueeze(-1))
return x.squeeze(-1)
 class PositionalEncoding(nn.Module):
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self ).__init__()
if ENC_DROP:
self.dropout = nn.Dropout(p=ENC_DROP)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
if ENC_DROP:
x = self.dropout(x)
 return x
class TransformerWrapper(nn.Module):
def __init__(self, dmodel=256, nhead=8, num_layers=2):
super(TransformerWrapper, self ).__init__()
self.pos_encoder = PositionalEncoding(256)
 encoder_layer = TransformerEncoderLayer(d_model=dmodel, nhead=nhead)
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.pos_emb = PositionalEncoding(dmodel)
 def flatten_parameters(self):
pass
def forward(self, x):
x = x.permute(( 1, 0, 2)).contiguous()
x = self.pos_emb(x)
 x = self.transformer_encoder(x)
 x = x.permute(( 1, 0, 2)).contiguous()
return x, None
class RnnLayers(nn.Module):
def __init__(self, dmodel, transformer_layers: int = 2):
super(RnnLayers, self ).__init__()
self.dropout = nn.Dropout(RNN_DROP)
 self.rnn0 = TransformerWrapper(dmodel, nhead=8, num_layers=transformer_layers)
 self.rnn1 = nn.LSTM(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 self.rnn2 = nn.GRU(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 def forward(self, x):
self.rnn0.flatten_parameters()
x, _ = self.rnn0(x)
 if self.rnn1 is not None:
self.rnn1.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn1(x)
 if self.rnn2 is not None:
self.rnn2.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn2(x)
 return x
class BaseAttnModel(nn.Module):
def __init__(self, transformer_layers=2, d=256):
super(BaseAttnModel, self ).__init__()
if USE_FT:
self.linear0 = nn.Linear(14 + 3, 1)
 self.seq_encoder_x = SeqEncoder(in_dim=18, out_dim=d)
 else:
self.linear0 = nn.Linear(14, 1)
 self.seq_encoder_x = SeqEncoder(in_dim=15, out_dim=d)
 self.attn = BppAttn(d, d//2)
 self.seq_encoder_bpp = SeqEncoder(in_dim=d//2, out_dim=d)
 self.seq = RnnLayers(d * 2, transformer_layers=transformer_layers)
 def forward(self, x, bpp):
bpp_features = get_bpp_feature(bpp[:, :, :, 0].float())
 if USE_FT:
x = torch.cat([x] + bpp_features, dim=-1)
 learned = self.linear0(x)
 x = torch.cat([x, learned], dim=-1)
 x = x.permute(0, 2, 1 ).contiguous().float()
bpp = bpp.permute([0, 3, 1, 2] ).contiguous().float()
x = self.seq_encoder_x(x)
 bpp = self.attn(x, bpp)
 bpp = self.seq_encoder_bpp(bpp)
 x = x.permute(0, 2, 1 ).contiguous()
bpp = bpp.permute(0, 2, 1 ).contiguous()
x = torch.cat([x, bpp], dim=2)
 x = self.seq(x)
 return x
class AEModel(nn.Module):
def __init__(self, transformer_layers=2):
super(AEModel, self ).__init__()
self.seq = BaseAttnModel(transformer_layers=transformer_layers, d=D)
 self.linear = nn.Sequential(
 nn.Linear(D * 2, 14),
nn.Sigmoid() , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = F.dropout(x, p=0.3)
 x = self.linear(x)
 return x
class FromAeModel(nn.Module):
def __init__(self, seq):
super(FromAeModel, self ).__init__()
self.seq = seq
self.linear = nn.Sequential(
 nn.Linear(D * 2, len(target_cols)) , 
)
 def forward(self, x, bpp, pred_len=68):
x = self.seq(x, bpp)
 x = self.linear(x)
 x = x[:, :pred_len]
return x 
<define_variables>
test_df['Survived'] = None
all_df = pd.concat([train_df, test_df], axis=0 )
PRETRAIN = False<create_dataframe>
def encrypt_single_column(data):
le = LabelEncoder()
le.fit(data.astype(str))
return le.transform(data.astype(str))
features, _ = preprocess(train_df, True)
 features_tensor = torch.from_numpy(features)
 dataset0 = VacDataset(features_tensor, train_df, None)
 features, _ = preprocess(public_df, True)
 features_tensor = torch.from_numpy(features)
 dataset1 = VacDataset(features_tensor, public_df, None)
 features, _ = preprocess(private_df, True)
 features_tensor = torch.from_numpy(features)
 dataset2 = VacDataset(features_tensor, private_df, None )<load_pretrained>
features = ['Pclass','Sex','Embarked','SibSp','Parch']
for feature in features:
all_df[feature] = encrypt_single_column(all_df[feature] )
BATCH_SIZE = 64
loader0 = torch.utils.data.DataLoader(dataset0, BATCH_SIZE, shuffle=True)
 loader1 = torch.utils.data.DataLoader(dataset1, BATCH_SIZE, shuffle=True)
 loader2 = torch.utils.data.DataLoader(dataset2, BATCH_SIZE, shuffle=True )<find_best_model_class>
X = all_df.loc[~(all_df.Fare.isna())]
y = X['Fare'].values
X = X[features]
X_test = all_df.loc[all_df.Fare.isna() ]
X_test = X_test[features]
print(f'X: {X.shape} y: {y.shape}, X_text: {X_test.shape}' )
def learn_from_batch_ae(model, data):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(DEVICE), data["bpp"].to(DEVICE))
loss = F.binary_cross_entropy(out, target.to(DEVICE))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, start_epoch=0, start_it=0, log_path="./logs"):
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in range(start_epoch, end_epoch):
model.train()
losses = []
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'Epoch {epoch} \t loss={loss_m:.4f}')
 torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
<define_variables>
clf = DecisionTreeRegressor()
clf.fit(X, y)
 y_test = clf.predict(X_test )
set_seed(SEED )<categorify>
print(f'Fare: {y_test}' )
if PRETRAIN:
model = AEModel()
model = model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
print(' -> Training with train data')
 res = train_ae(model, loader0, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 print('
-> Training with public data')
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 print('
-> Training with private data')
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, start_epoch=res["end_epoch"], start_it=res["it"])
 save_model_weights(model, CP_PATH + f'pretrained_model_{TODAY}.pt')
 print('
Done.' )<define_search_space>
all_df.loc[all_df.Fare.isna() , 'Fare'] = y_test
CLASS_WEIGHT_5 = torch.from_numpy(np.array([1, 1, 1, 1, 1])).unsqueeze(0 ).cuda()
CLASS_WEIGHT_3 = torch.from_numpy(np.array([1, 1, 0, 1, 0])).unsqueeze(0 ).cuda()<compute_test_metric>
all_df.loc[all_df.Fare.isna() ].shape
def mcrmse(truth, pred, verbose=0, scored_targets=[0, 1, 3], filtered=None, reduce=True):

 error =(truth - pred)** 2
error = error[:, :, scored_targets]
if filtered is not None:
error = np.array([error[i] for i, kept in enumerate(filtered)if kept])
 rmse = np.sqrt(error.mean(1))
if verbose:
for t, score in zip(scored_targets, rmse.mean(0)) :
print(f'Score for target "{TARGETS[t]}":\t {score:.4f}')
 if reduce:
return rmse.mean()
else:
return rmse.mean(-1)
 def MCRMSE(y_true, y_pred, class_weight):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse)* class_weight, dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise, class_weight):
loss = MCRMSE(target, predict, class_weight)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 weight = torch.clamp(weight, 0.01, 1000)
 loss =(loss * weight ).mean()
return loss<choose_model_class>
all_df = [train_df, test_df]
def learn_from_batch(model, data, optimizer, lr_scheduler, class_weight, pred_len=68):
optimizer.zero_grad()
out = model(
 data["sequence"].to(DEVICE),
data["bpp"].to(DEVICE),
pred_len=pred_len, 
)
 signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(
 out,
data["label"].to(DEVICE),
signal_to_noise.to(DEVICE),
class_weight, 
)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, class_weight):
model.eval()
loss_list = []
mcrmses = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(
 data["sequence"].to(DEVICE),
data["bpp"].to(DEVICE),
pred_len=68, 
)
 mcrmse_ = mcrmse(data["label"].numpy() , y.cpu().numpy() , filtered=data["signal_to_noise"] > 1, reduce=False)
 mcrmses += list(mcrmse_)
 loss = sn_mcrmse_loss(
 y,
data["label"].to(DEVICE),
data["signal_to_noise"].to(DEVICE),
class_weight 
)
 loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcrmse=np.mean(mcrmses))<predict_on_test>
for dataset in all_df:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False )
def predict(model, loader, pred_len=68):

 model.eval()
preds = np.empty(( 0, pred_len, NUM_TARGETS))
with torch.no_grad() :
for batch in loader:
y_pred = model(
 batch["sequence"].cuda() ,
batch["bpp"].cuda() ,
pred_len=pred_len,
).detach()
preds = np.concatenate([preds, y_pred.cpu().numpy() ])
 return preds<train_model>
for dataset in all_df:
dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs' )
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, swa_first_epoch=40, class_weight=None, pred_len=68):
it = 0
for epoch in range(epochs):
t0 = time.time()
print(f"Epoch {epoch+1}/{epochs}", end='\t')
 model.train()
losses = []
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, class_weight, pred_len=pred_len)
 loss_v = loss.item()
losses.append(loss_v)
 it += 1
if epoch + 1 >= swa_first_epoch:
optimizer.update_swa()
optimizer.swap_swa_sgd()
eval_result = evaluate(model, valid_data, class_weight)
 eval_loss = eval_result["loss"]
if epoch + 1 >= swa_first_epoch and epoch < epochs - 1:
optimizer.swap_swa_sgd()
dt = time.time() - t0
lr = lr_scheduler.get_last_lr() [0] if lr_scheduler else 1e-3
print(f't={dt:.1f}s\t lr={lr:.1e}\tloss={np.mean(losses):.4f}', end='\t')
 print(f"val_loss={eval_loss:.4f} \t val_mcrmse={eval_result['mcrmse']:.4f}" )<define_variables>
train_df[(train_df['Title'] == 'Dr')&(train_df['Sex'] == 'female')]
EPOCHS_1 = 30
EPOCHS_2 = 10
EPOCHS_3 = 10
EPOCHS_4 = 5
SWA_FIRST_EPOCH = 0
WARMUP_PROP = 0.05
K = 5
BATCH_SIZE = 32
LR = 5e-4
LOAD = True<drop_column>
train_df[train_df['Cabin']=='D17']
samples = train_df.copy().drop('score', axis=1)
 ids = samples.reset_index() ["id"]
set_seed(SEED )<split>
train_df.loc[train_df.PassengerId == 797, 'Title'] = 'Mrs'
gkf = GroupKFold(n_splits=K)
 splits = list(gkf.split(X=samples, groups=groups))<categorify>
train_df[train_df['Cabin']=='D17']
scores = []
pred_oof = np.zeros(( len(samples), 68, NUM_TARGETS))
for fold,(train_index, test_index)in enumerate(splits):
print(f"
------------- Fold {fold + 1}/{K} -------------
")
 set_seed(SEED)
 df_train = samples.loc[train_index].reset_index()
df_train = augment_data(df_train)
 df_val = samples.loc[test_index].reset_index()
df_val_aug = augment_data(df_val.copy() , concat=False)
 train_loader = create_loader(df_train, BATCH_SIZE)
 public_loader = create_loader(public_df, BATCH_SIZE)
 private_loader = create_loader(private_df, BATCH_SIZE)
 valid_loader = create_loader(df_val, BATCH_SIZE, shuffle=False)
 valid_loader_aug = create_loader(df_val_aug, BATCH_SIZE, shuffle=False)
 ae_model = AEModel()
if LOAD:
load_model_weights(ae_model, PRETRAINED_PATH)
 model = FromAeModel(ae_model.seq)
 model = model.to(DEVICE)
 model.zero_grad()
model.train()
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 if SWA_FIRST_EPOCH < EPOCHS_4:
optimizer = SWA(optimizer)
 s1 = EPOCHS_1 * len(train_loader)
 s2 = EPOCHS_2 * len(public_loader)
 s3 = EPOCHS_3 * len(private_loader)
 s4 = EPOCHS_4 * len(train_loader)
 num_training_steps = int(s1 + s2 + s3 + s4)
 num_warmup_steps = int(EPOCHS_1 * len(train_loader)* WARMUP_PROP)
 lr_scheduler = get_linear_schedule_with_warmup(
 optimizer, num_warmup_steps, num_training_steps 
)
 print('
-> Training with train data
')
 train(
 model, train_loader, valid_loader, optimizer, lr_scheduler, pred_len=68,
epochs=EPOCHS_1, swa_first_epoch=100, class_weight=CLASS_WEIGHT_5)
 print('
-> Training with public data
')
 train(
 model, public_loader, valid_loader, optimizer, lr_scheduler, pred_len=SEQ_SCORED_PUBLIC,
epochs=EPOCHS_2, swa_first_epoch=100, class_weight=CLASS_WEIGHT_3 
)
 print('
-> Training with private data
')
 train(
 model, private_loader, valid_loader, optimizer, lr_scheduler, pred_len=SEQ_SCORED_PRIVATE,
epochs=EPOCHS_3, swa_first_epoch=100, class_weight=CLASS_WEIGHT_3 
)
 print('
-> Retraining with train data
')
 train(
 model, train_loader, valid_loader, optimizer, lr_scheduler, pred_len=68,
epochs=EPOCHS_4, swa_first_epoch=SWA_FIRST_EPOCH, class_weight=CLASS_WEIGHT_3 
)
 y = np.array(df_val[TARGETS].values.tolist() ).transpose(( 0, 2, 1))
pred_val_ = predict(model, valid_loader)
 pred_val_aug = predict(model, valid_loader_aug)
 pred_val =(0.5 * pred_val_ + 0.5 * pred_val_aug)
 pred_oof[test_index] = pred_val
score = mcrmse(y, pred_val, verbose=0, scored_targets=[0, 1, 3], filtered=df_val["signal_to_noise"] > 1)
 scores.append(score)
 print(f"
-> Scored {score:.4f} with TTA")
 save_model_weights(model, f'model_{fold}.pt', verbose=0, cp_folder=CP_PATH)
 del model<define_variables>
for dataset in all_df:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare' )
BATCH_SIZE = 64
TTA = True<load_pretrained>
train_df[['Title', 'Sex', 'Survived']].groupby(['Title', 'Sex'], as_index=False ).mean()
test_df = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = test_df[test_df["seq_length"] == SEQ_LEN_PUBLIC].reset_index(drop=True)
 private_df = test_df[test_df["seq_length"] == SEQ_LEN_PRIVATE].reset_index(drop=True)
 pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 public_df_aug = augment_data(public_df.copy().drop('score', axis=1), concat=False)
 pub_loader_aug = create_loader(public_df_aug, BATCH_SIZE, is_test=True)
 private_df_aug = augment_data(private_df.copy().drop('score', axis=1), concat=False)
 pri_loader_aug = create_loader(private_df_aug, BATCH_SIZE, is_test=True )<categorify>
for dataset in all_df:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
pred_df_list = []
pred_public = np.zeros(( len(public_df), 107, NUM_TARGETS))
pred_private = np.zeros(( len(private_df), SEQ_LEN_PRIVATE, NUM_TARGETS))
for fold in range(K):
print(f"
------------- Fold {fold + 1}/{K} -------------
")
 model_load_path = CP_PATH + f"model_{fold}.pt"
print(f' -> Loading weights from {model_load_path}
')
 ae_model0 = AEModel()
model_pub = FromAeModel(seq=ae_model0.seq)
 model_pub = model_pub.to(DEVICE)
 ae_model1 = AEModel()
model_pri = FromAeModel(seq=ae_model1.seq)
 model_pri = model_pri.to(DEVICE)
 state_dict = torch.load(model_load_path, map_location=DEVICE)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
pred_public += predict(model_pub, pub_loader, pred_len=107)/ K
pred_private += predict(model_pri, pri_loader, pred_len=130)/ K
if TTA:
pred_public += predict(model_pub, pub_loader_aug, pred_len=107)/ K
pred_private += predict(model_pri, pri_loader_aug, pred_len=130)/ K
if TTA:
pred_public *= 0.5
pred_private *= 0.5<compute_test_metric>
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean()
def pred_to_sub(df_test, pred_public, pred_private):
sub_public = df_test[df_test['seq_scored'] == SEQ_SCORED_PUBLIC][['id']].reset_index(drop=True)
 sub_private = df_test[df_test['seq_scored'] == SEQ_SCORED_PRIVATE][['id']].reset_index(drop=True)
 test_preds = []
for sub, pred in [(sub_public, pred_public),(sub_private, pred_private)]:
for i, seq in enumerate(sub.id):
single_pred = pred[i]
single_df = pd.DataFrame(single_pred, columns=TARGETS)
 single_df['id_seqpos'] = [f'{seq}_{x}' for x in range(single_df.shape[0])]
test_preds.append(single_df)
 return pd.concat(test_preds)[['id_seqpos'] + TARGETS]<compute_test_metric>
for dataset in all_df:
dataset['Surname'] = dataset.Name.str.extract('([A-Za-z]+)\,', expand=False )
sub = pred_to_sub(test_df, pred_public, pred_private )<compute_test_metric>
tmp = train_df.groupby(['Surname'])['Survived'].value_counts()
df = pd.DataFrame(data={'Size of group with same Surname': tmp.values}, index=tmp.index ).reset_index().sort_values(['Size of group with same Surname', 'Surname'], ascending=False )
score = np.mean(scores)
 score<save_to_csv>
tmp = df.groupby(['Size of group with same Surname'])['Survived'].value_counts()
df = pd.DataFrame(data={'Number': tmp.values}, index=tmp.index ).reset_index().sort_values(['Size of group with same Surname', 'Survived'], ascending=False)
 df
print(f'Saving submission to "{TODAY}_{score:.4f}_pl.csv"')
 sub.to_csv(f"{TODAY}_{score:.4f}_pl.csv", index=False)
 np.save(f"oof_{TODAY}_{score:.4f}_pl.npy", pred_oof)
 sub.head()<import_modules>
for dataset in all_df:
dataset['Deck'] = dataset.Cabin.str.extract('^([A-Za-z]+)', expand=False )
import numpy as np
import pandas as pd
import os<load_from_csv>
train_df[['Deck', 'Survived']].groupby(['Deck'], as_index=False ).mean()
weight_aepgc = 0.27625
sub1 = pd.read_csv('.. /input/24551-ae-gcn/submission(3 ).csv')
 sub2 = pd.read_csv('.. /input/hawkey-ae-pretrained-gcn-3ensemble/submission(4 ).csv' )<load_from_csv>
for dataset in all_df:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int )
weight_nprg = 0.27625
sub3 = pd.read_csv('.. /input/gru-5fold-2seeds-knncv-68c7e1/submission.csv')
 sub4 = pd.read_csv('.. /input/fork-of-lstm-gru-5fold-2seeds-knncv-0a37d7/submission.csv')
 sub5 = pd.read_csv('.. /input/lstm-5fold-2seeds-knncv-775f37/submission.csv')
 sub6 = pd.read_csv('.. /input/hawkey-gcn-only-25301/submission(5 ).csv')
 sub7 = pd.read_csv('.. /input/wavenet-final/submission.csv')
 sub8 = pd.read_csv('.. /input/wavenet2-moa/wavenet2_249.csv' )<load_from_csv>
age_aprox = np.zeros(( 2,3))
for dataset in all_df:
for i in range(0, 2):
for j in range(0, 3):
aprox_age = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_aprox[i,j] = aprox_age.median()
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),'Age'] = \
age_aprox[i,j]
dataset['Age'] = dataset['Age'].astype(int )
weight_aug = 0.0975
sub9 = pd.read_csv('.. /input/aug-data-local-training-gru-lstm/aug_data_training_local.csv' )<load_from_csv>
tmp = train_df.groupby(['Title', 'Pclass'])['Survived'].value_counts()
df = pd.DataFrame(data={'Passengers': tmp.values}, index=tmp.index ).reset_index()
df
weight_pt = 0.35
sub10 = pd.read_csv('.. /input/open-vaccine-pytorch-pretrain/submission.csv')
 sub11 = pd.read_csv('.. /input/4-ae-pretrained-sin/submission_4_ae.csv' )<load_from_csv>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in all_df:
dataset['Title'] = dataset['Title'].map(title_mapping )
sub12 = pd.read_csv('.. /input/blend-of-public/blend_of_public.csv' )<load_from_csv>
for dataset in all_df:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
final = pd.read_csv('.. /input/stanford-covid-vaccine/sample_submission.csv' ).set_index('id_seqpos')
 sample_sub = pd.read_csv('.. /input/stanford-covid-vaccine/sample_submission.csv' )<filter>
for dataset in all_df:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
sub1 = sub1[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub2 = sub2[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub3 = sub3[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub4 = sub4[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub5 = sub5[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub6 = sub6[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub7 = sub7[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub8 = sub8[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub9 = sub9[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub10 = sub10[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub11 = sub11[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]
sub12 = sub12[sample_sub.columns].set_index('id_seqpos' ).loc[final.index]<compute_test_metric>
for dataset in all_df:
dataset.loc[ dataset['FamilySize'] <= 1, 'FamilySize'] = 0
dataset.loc[(dataset['FamilySize'] > 1)&(dataset['FamilySize'] <= 4), 'FamilySize'] = 1
dataset.loc[ dataset['FamilySize'] > 4, 'FamilySize'] = 2
final =(( sub1 + sub2)/ 2 * weight_aepgc +(( sub3 + sub4 + sub5 + sub6 + sub7)/ 5 * 0.85 + sub8 * 0.15)* weight_nprg + sub9 * weight_aug +(sub10 * 0.7 + sub11 * 0.3)* weight_pt)*.93 + sub12 *.07<save_to_csv>
for dataset in all_df:
dataset['Class*Age'] = dataset['Pclass'] * dataset['Age']
final.to_csv('final_sub_nopp.csv' )<define_variables>
VALID_SIZE = 0.2
RANDOM_STATE = 2018
train, valid = train_test_split(train_df, test_size=VALID_SIZE, random_state=RANDOM_STATE, shuffle=True )
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
input_cols = ['sequence', 'structure', 'predicted_loop_type']
error_cols = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_Mg_50C', 'deg_error_pH10', 'deg_error_50C']
token_dicts = {
"sequence": {x: i for i, x in enumerate("ACGU")},
"structure": {x: i for i, x in enumerate('().')},
"predicted_loop_type": {x: i for i, x in enumerate("BEHIMSX")}
}
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed )<define_variables>
predictors = ['Sex', 'Age']
target = 'Survived'
BASE_PATH = "/kaggle/input/stanford-covid-vaccine"
MODEL_SAVE_PATH = "/kaggle/model"
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 print(Ds.shape)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss
def create_loader(df, batch_size=1, is_test=False):
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, drop_last=False)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False, drop_last=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index])
<choose_model_class>
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class Conv2dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv2dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv2d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, 128, 3, padding=1)
 self.conv1 = Conv1dStack(128, 64, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(64, 32, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(32, 32, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class BppAttn(nn.Module):
def __init__(self, in_channel: int, out_channel: int):
super(BppAttn, self ).__init__()
self.conv0 = Conv1dStack(in_channel, out_channel, 3, padding=1)
 self.bpp_conv = Conv2dStack(5, out_channel)
 def forward(self, x, bpp):
x = self.conv0(x)
 bpp = self.bpp_conv(bpp)
 x = torch.matmul(bpp, x.unsqueeze(-1))
return x.squeeze(-1)
 class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x)
 class TransformerWrapper(nn.Module):
def __init__(self, dmodel=256, nhead=8, num_layers=2):
super(TransformerWrapper, self ).__init__()
self.pos_encoder = PositionalEncoding(256)
 encoder_layer = TransformerEncoderLayer(d_model=dmodel, nhead=nhead)
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.pos_emb = PositionalEncoding(dmodel)
 def flatten_parameters(self):
pass
def forward(self, x):
x = x.permute(( 1, 0, 2)).contiguous()
x = self.pos_emb(x)
 x = self.transformer_encoder(x)
 x = x.permute(( 1, 0, 2)).contiguous()
return x, None
class RnnLayers(nn.Module):
def __init__(self, dmodel, dropout=0.3, transformer_layers: int = 2):
super(RnnLayers, self ).__init__()
self.dropout = nn.Dropout(dropout)
 self.rnn0 = TransformerWrapper(dmodel, nhead=8, num_layers=transformer_layers)
 self.rnn1 = nn.LSTM(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 self.rnn2 = nn.GRU(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 def forward(self, x):
self.rnn0.flatten_parameters()
x, _ = self.rnn0(x)
 if self.rnn1 is not None:
self.rnn1.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn1(x)
 if self.rnn2 is not None:
self.rnn2.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn2(x)
 return x
class BaseAttnModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(BaseAttnModel, self ).__init__()
self.linear0 = nn.Linear(14 + 3, 1)
 self.seq_encoder_x = SeqEncoder(18)
 self.attn = BppAttn(256, 128)
 self.seq_encoder_bpp = SeqEncoder(128)
 self.seq = RnnLayers(256 * 2, dropout=0.3,
transformer_layers=transformer_layers)
 def forward(self, x, bpp):
bpp_features = get_bpp_feature(bpp[:, :, :, 0].float())
 x = torch.cat([x] + bpp_features, dim=-1)
 learned = self.linear0(x)
 x = torch.cat([x, learned], dim=-1)
 x = x.permute(0, 2, 1 ).contiguous().float()
bpp = bpp.permute([0, 3, 1, 2] ).contiguous().float()
x = self.seq_encoder_x(x)
 bpp = self.attn(x, bpp)
 bpp = self.seq_encoder_bpp(bpp)
 x = x.permute(0, 2, 1 ).contiguous()
bpp = bpp.permute(0, 2, 1 ).contiguous()
x = torch.cat([x, bpp], dim=2)
 x = self.seq(x)
 return x
class AEModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(AEModel, self ).__init__()
self.seq = BaseAttnModel(transformer_layers=transformer_layers)
 self.linear = nn.Sequential(
 nn.Linear(256 * 2, 14),
nn.Sigmoid() , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = F.dropout(x, p=0.3)
 x = self.linear(x)
 return x
class FromAeModel(nn.Module):
def __init__(self, seq, pred_len=68, dmodel: int = 256):
super(FromAeModel, self ).__init__()
self.seq = seq
self.pred_len = pred_len
self.linear = nn.Sequential(
 nn.Linear(dmodel * 2, len(target_cols)) , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = self.linear(x)
 x = x[:, :self.pred_len]
return x 
<load_from_disk>
RFC_METRIC = 'gini'
NUM_ESTIMATORS = 100
NO_JOBS = 4
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_train_data.head()
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
features, _ = preprocess(base_train_data, True)
 features_tensor = torch.from_numpy(features)
 dataset0 = VacDataset(features_tensor, base_train_data, None)
 features, _ = preprocess(public_df, True)
 features_tensor = torch.from_numpy(features)
 dataset1 = VacDataset(features_tensor, public_df, None)
 features, _ = preprocess(private_df, True)
 features_tensor = torch.from_numpy(features)
 dataset2 = VacDataset(features_tensor, private_df, None)
 loader0 = torch.utils.data.DataLoader(dataset0, BATCH_SIZE, shuffle=False, drop_last=False)
 loader1 = torch.utils.data.DataLoader(dataset1, BATCH_SIZE, shuffle=False, drop_last=False)
 loader2 = torch.utils.data.DataLoader(dataset2, BATCH_SIZE, shuffle=False, drop_last=False )<find_best_model_class>
clf = RandomForestClassifier(n_jobs=NO_JOBS,
random_state=RANDOM_STATE,
criterion=RFC_METRIC,
n_estimators=NUM_ESTIMATORS,
verbose=False )
def learn_from_batch_ae(model, data, device):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(device), data["bpp"].to(device))
loss = F.binary_cross_entropy(out, target.to(device))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, start_it: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data, device)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'epoch: {epoch} loss: {loss_m}')
 losses = []
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
<choose_model_class>
clf.fit(train_X, train_Y )
set_seed(123)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 lr_scheduler = None
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AEModel()
model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
res = train_ae(model, loader0, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 epoch = res["min_loss_epoch"]
shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"model-{epoch}.pt"), "ae-model.pt" )<compute_test_metric>
preds = clf.predict(valid_X )
def MCRMSE(y_true, y_pred):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse), dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise):
loss = MCRMSE(target, predict)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 loss =(loss * weight ).mean()
return loss
def learn_from_batch(model, data, optimizer, lr_scheduler, device):
optimizer.zero_grad()
out = model(data["sequence"].to(device), data["bpp"].to(device))
signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(out, data["label"].to(device), signal_to_noise.to(device))
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, device):
model.eval()
loss_list = []
mcrmse = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(data["sequence"].to(device), data["bpp"].to(device))
mcrmse_ = MCRMSE(data["label"].to(device), y)[data["signal_to_noise"] > 1]
mcrmse.append(mcrmse_.mean().item())
 loss = sn_mcrmse_loss(y, data["label"].to(device), data["signal_to_noise"].to(device))
loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcmse=np.mean(mcrmse))
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
writer = SummaryWriter(log_path)
 it = 0
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 min_eval_loss = 10.0
min_eval_epoch = None
for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, device)
 loss_v = loss.item()
writer.add_scalar('loss', loss_v, it)
 losses.append(loss_v)
 it += 1
print(f'epoch: {epoch} loss: {np.mean(losses)}')
 losses = []
eval_result = evaluate(model, valid_data, device)
 eval_loss = eval_result["loss"]
if eval_loss <= min_eval_loss:
min_eval_epoch = epoch
min_eval_loss = eval_loss
print(f"eval loss: {eval_loss} {eval_result['mcmse']}")
 writer.add_scalar(f"evaluate/loss", eval_loss, epoch)
 writer.add_scalar(f"evaluate/mcmse", eval_result["mcmse"], epoch)
 model.train()
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
print(f'min eval loss: {min_eval_loss} epoch {min_eval_epoch}')
 return min_eval_epoch 
<load_pretrained>
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 samples = base_train_data
save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 split = ShuffleSplit(n_splits=10, test_size=.1)
 ids = samples.reset_index() ["id"]
set_seed(124)
 for fold,(train_index, test_index)in enumerate(split.split(samples)) :
print(f"fold: {fold}")
 train_df = samples.loc[train_index].reset_index()
val_df = samples.loc[test_index].reset_index()
train_loader = create_loader(train_df, BATCH_SIZE)
 valid_loader = create_loader(val_df, BATCH_SIZE)
 print(train_df.shape, val_df.shape)
 ae_model = AEModel()
state_dict = torch.load("./ae-model.pt")
 ae_model.load_state_dict(state_dict)
 del state_dict
model = FromAeModel(ae_model.seq)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
epoch = train(model, train_loader, valid_loader, optimizer, lr_scheduler, 200, device=device,
log_path=f"logs/{fold}")
 shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"./model-{epoch}.pt"), f"model_prediction/model-{fold}.pt")
 del model<predict_on_test>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
def predict_batch(model, data, device):
with torch.no_grad() :
pred = model(data["sequence"].to(device), data["bpp"].to(device))
pred = pred.detach().cpu().numpy()
return_values = []
ids = data["ids"]
for idx, p in enumerate(pred):
id_ = ids[idx]
assert p.shape ==(model.pred_len, len(target_cols))
for seqpos, val in enumerate(p):
assert len(val)== len(target_cols)
 dic = {key: val for key, val in zip(target_cols, val)}
dic["id_seqpos"] = f"{id_}_{seqpos}"
return_values.append(dic)
 return return_values
def predict_data(model, loader, device, batch_size):
data_list = []
for i, data in enumerate(progress_bar(loader)) :
data_list += predict_batch(model, data, device)
 expected_length = model.pred_len * len(loader)* batch_size
assert len(data_list)== expected_length, f"len = {len(data_list)} expected = {expected_length}"
return data_list 
<load_pretrained>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
device = torch.device('cuda')if torch.cuda.is_available() else "cpu"
BATCH_SIZE = 1
base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 pred_df_list = []
c = 0
for fold in range(10):
model_load_path = f"./model_prediction/model-{fold}.pt"
ae_model0 = AEModel()
ae_model1 = AEModel()
model_pub = FromAeModel(pred_len=107, seq=ae_model0.seq)
 model_pub = model_pub.to(device)
 model_pri = FromAeModel(pred_len=130, seq=ae_model1.seq)
 model_pri = model_pri.to(device)
 state_dict = torch.load(model_load_path, map_location=device)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
data_list = []
data_list += predict_data(model_pub, pub_loader, device, BATCH_SIZE)
 data_list += predict_data(model_pri, pri_loader, device, BATCH_SIZE)
 pred_df = pd.DataFrame(data_list, columns=["id_seqpos"] + target_cols)
 print(pred_df.head())
 print(pred_df.tail())
 pred_df_list.append(pred_df)
 c += 1
data_dic = dict(id_seqpos=pred_df_list[0]["id_seqpos"])
 for col in target_cols:
vals = np.zeros(pred_df_list[0][col].shape[0])
 for df in pred_df_list:
vals += df[col].values
data_dic[col] = vals / float(c)
 pred_df_avg = pd.DataFrame(data_dic, columns=["id_seqpos"] + target_cols)
 print(pred_df_avg.head())
 pred_df_avg.to_csv("./submission.csv", index=False)
<import_modules>
predictors = ['Sex', 'Age', 'Pclass', 'Fare']
target = 'Survived'
import numpy as np
import pandas as pd
import os<define_variables>
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
input_cols = ['sequence', 'structure', 'predicted_loop_type']
error_cols = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_Mg_50C', 'deg_error_pH10', 'deg_error_50C']
token_dicts = {
"sequence": {x: i for i, x in enumerate("ACGU")},
"structure": {x: i for i, x in enumerate('().')},
"predicted_loop_type": {x: i for i, x in enumerate("BEHIMSX")}
}
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed )<define_variables>
clf.fit(train_X, train_Y )
BASE_PATH = "/kaggle/input/stanford-covid-vaccine"
MODEL_SAVE_PATH = "/kaggle/model"
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 print(Ds.shape)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss
def create_loader(df, batch_size=1, is_test=False):
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, drop_last=False)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False, drop_last=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index])
<choose_model_class>
preds = clf.predict(valid_X )
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class Conv2dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv2dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv2d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm2d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, 128, 3, padding=1)
 self.conv1 = Conv1dStack(128, 64, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(64, 32, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(32, 32, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class BppAttn(nn.Module):
def __init__(self, in_channel: int, out_channel: int):
super(BppAttn, self ).__init__()
self.conv0 = Conv1dStack(in_channel, out_channel, 3, padding=1)
 self.bpp_conv = Conv2dStack(5, out_channel)
 def forward(self, x, bpp):
x = self.conv0(x)
 bpp = self.bpp_conv(bpp)
 x = torch.matmul(bpp, x.unsqueeze(-1))
return x.squeeze(-1)
 class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x)
 class TransformerWrapper(nn.Module):
def __init__(self, dmodel=256, nhead=8, num_layers=2):
super(TransformerWrapper, self ).__init__()
self.pos_encoder = PositionalEncoding(256)
 encoder_layer = TransformerEncoderLayer(d_model=dmodel, nhead=nhead)
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.pos_emb = PositionalEncoding(dmodel)
 def flatten_parameters(self):
pass
def forward(self, x):
x = x.permute(( 1, 0, 2)).contiguous()
x = self.pos_emb(x)
 x = self.transformer_encoder(x)
 x = x.permute(( 1, 0, 2)).contiguous()
return x, None
class RnnLayers(nn.Module):
def __init__(self, dmodel, dropout=0.3, transformer_layers: int = 2):
super(RnnLayers, self ).__init__()
self.dropout = nn.Dropout(dropout)
 self.rnn0 = TransformerWrapper(dmodel, nhead=8, num_layers=transformer_layers)
 self.rnn1 = nn.LSTM(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 self.rnn2 = nn.GRU(dmodel, dmodel // 2, batch_first=True, num_layers=1, bidirectional=True)
 def forward(self, x):
self.rnn0.flatten_parameters()
x, _ = self.rnn0(x)
 if self.rnn1 is not None:
self.rnn1.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn1(x)
 if self.rnn2 is not None:
self.rnn2.flatten_parameters()
x = self.dropout(x)
 x, _ = self.rnn2(x)
 return x
class BaseAttnModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(BaseAttnModel, self ).__init__()
self.linear0 = nn.Linear(14 + 3, 1)
 self.seq_encoder_x = SeqEncoder(18)
 self.attn = BppAttn(256, 128)
 self.seq_encoder_bpp = SeqEncoder(128)
 self.seq = RnnLayers(256 * 2, dropout=0.3,
transformer_layers=transformer_layers)
 def forward(self, x, bpp):
bpp_features = get_bpp_feature(bpp[:, :, :, 0].float())
 x = torch.cat([x] + bpp_features, dim=-1)
 learned = self.linear0(x)
 x = torch.cat([x, learned], dim=-1)
 x = x.permute(0, 2, 1 ).contiguous().float()
bpp = bpp.permute([0, 3, 1, 2] ).contiguous().float()
x = self.seq_encoder_x(x)
 bpp = self.attn(x, bpp)
 bpp = self.seq_encoder_bpp(bpp)
 x = x.permute(0, 2, 1 ).contiguous()
bpp = bpp.permute(0, 2, 1 ).contiguous()
x = torch.cat([x, bpp], dim=2)
 x = self.seq(x)
 return x
class AEModel(nn.Module):
def __init__(self, transformer_layers: int = 2):
super(AEModel, self ).__init__()
self.seq = BaseAttnModel(transformer_layers=transformer_layers)
 self.linear = nn.Sequential(
 nn.Linear(256 * 2, 14),
nn.Sigmoid() , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = F.dropout(x, p=0.3)
 x = self.linear(x)
 return x
class FromAeModel(nn.Module):
def __init__(self, seq, pred_len=68, dmodel: int = 256):
super(FromAeModel, self ).__init__()
self.seq = seq
self.pred_len = pred_len
self.linear = nn.Sequential(
 nn.Linear(dmodel * 2, len(target_cols)) , 
)
 def forward(self, x, bpp):
x = self.seq(x, bpp)
 x = self.linear(x)
 x = x[:, :self.pred_len]
return x 
<load_from_disk>
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_train_data.head()
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
features, _ = preprocess(base_train_data, True)
 features_tensor = torch.from_numpy(features)
 dataset0 = VacDataset(features_tensor, base_train_data, None)
 features, _ = preprocess(public_df, True)
 features_tensor = torch.from_numpy(features)
 dataset1 = VacDataset(features_tensor, public_df, None)
 features, _ = preprocess(private_df, True)
 features_tensor = torch.from_numpy(features)
 dataset2 = VacDataset(features_tensor, private_df, None)
 loader0 = torch.utils.data.DataLoader(dataset0, BATCH_SIZE, shuffle=False, drop_last=False)
 loader1 = torch.utils.data.DataLoader(dataset1, BATCH_SIZE, shuffle=False, drop_last=False)
 loader2 = torch.utils.data.DataLoader(dataset2, BATCH_SIZE, shuffle=False, drop_last=False )<find_best_model_class>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
def learn_from_batch_ae(model, data, device):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(device), data["bpp"].to(device))
loss = F.binary_cross_entropy(out, target.to(device))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, start_it: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data, device)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'epoch: {epoch} loss: {loss_m}')
 losses = []
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
<choose_model_class>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
set_seed(123)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 lr_scheduler = None
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AEModel()
model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
res = train_ae(model, loader0, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 epoch = res["min_loss_epoch"]
shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"model-{epoch}.pt"), "ae-model.pt" )<compute_test_metric>
predictors = ['Sex', 'Age', 'Pclass', 'Fare', 'Parch', 'SibSp']
target = 'Survived'
def MCRMSE(y_true, y_pred):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse), dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise):
loss = MCRMSE(target, predict)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 loss =(loss * weight ).mean()
return loss
def learn_from_batch(model, data, optimizer, lr_scheduler, device):
optimizer.zero_grad()
out = model(data["sequence"].to(device), data["bpp"].to(device))
signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(out, data["label"].to(device), signal_to_noise.to(device))
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, device):
model.eval()
loss_list = []
mcrmse = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(data["sequence"].to(device), data["bpp"].to(device))
mcrmse_ = MCRMSE(data["label"].to(device), y)[data["signal_to_noise"] > 1]
mcrmse.append(mcrmse_.mean().item())
 loss = sn_mcrmse_loss(y, data["label"].to(device), data["signal_to_noise"].to(device))
loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcmse=np.mean(mcrmse))
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
writer = SummaryWriter(log_path)
 it = 0
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 min_eval_loss = 10.0
min_eval_epoch = None
for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, device)
 loss_v = loss.item()
writer.add_scalar('loss', loss_v, it)
 losses.append(loss_v)
 it += 1
print(f'epoch: {epoch} loss: {np.mean(losses)}')
 losses = []
eval_result = evaluate(model, valid_data, device)
 eval_loss = eval_result["loss"]
if eval_loss <= min_eval_loss:
min_eval_epoch = epoch
min_eval_loss = eval_loss
print(f"eval loss: {eval_loss} {eval_result['mcmse']}")
 writer.add_scalar(f"evaluate/loss", eval_loss, epoch)
 writer.add_scalar(f"evaluate/mcmse", eval_result["mcmse"], epoch)
 model.train()
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
print(f'min eval loss: {min_eval_loss} epoch {min_eval_epoch}')
 return min_eval_epoch 
<load_pretrained>
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 samples = base_train_data
save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 split = ShuffleSplit(n_splits=5, test_size=.1)
 ids = samples.reset_index() ["id"]
set_seed(124)
 for fold,(train_index, test_index)in enumerate(split.split(samples)) :
print(f"fold: {fold}")
 train_df = samples.loc[train_index].reset_index()
val_df = samples.loc[test_index].reset_index()
train_loader = create_loader(train_df, BATCH_SIZE)
 valid_loader = create_loader(val_df, BATCH_SIZE)
 print(train_df.shape, val_df.shape)
 ae_model = AEModel()
state_dict = torch.load("./ae-model.pt")
 ae_model.load_state_dict(state_dict)
 del state_dict
model = FromAeModel(ae_model.seq)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
epoch = train(model, train_loader, valid_loader, optimizer, lr_scheduler, 200, device=device,
log_path=f"logs/{fold}")
 shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"./model-{epoch}.pt"), f"model_prediction/model-{fold}.pt")
 del model<predict_on_test>
clf.fit(train_X, train_Y )
def predict_batch(model, data, device):
with torch.no_grad() :
pred = model(data["sequence"].to(device), data["bpp"].to(device))
pred = pred.detach().cpu().numpy()
return_values = []
ids = data["ids"]
for idx, p in enumerate(pred):
id_ = ids[idx]
assert p.shape ==(model.pred_len, len(target_cols))
for seqpos, val in enumerate(p):
assert len(val)== len(target_cols)
 dic = {key: val for key, val in zip(target_cols, val)}
dic["id_seqpos"] = f"{id_}_{seqpos}"
return_values.append(dic)
 return return_values
def predict_data(model, loader, device, batch_size):
data_list = []
for i, data in enumerate(progress_bar(loader)) :
data_list += predict_batch(model, data, device)
 expected_length = model.pred_len * len(loader)* batch_size
assert len(data_list)== expected_length, f"len = {len(data_list)} expected = {expected_length}"
return data_list 
<load_pretrained>
preds = clf.predict(valid_X )
device = torch.device('cuda')if torch.cuda.is_available() else "cpu"
BATCH_SIZE = 1
base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 pred_df_list = []
c = 0
for fold in range(5):
model_load_path = f"./model_prediction/model-{fold}.pt"
ae_model0 = AEModel()
ae_model1 = AEModel()
model_pub = FromAeModel(pred_len=107, seq=ae_model0.seq)
 model_pub = model_pub.to(device)
 model_pri = FromAeModel(pred_len=130, seq=ae_model1.seq)
 model_pri = model_pri.to(device)
 state_dict = torch.load(model_load_path, map_location=device)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
data_list = []
data_list += predict_data(model_pub, pub_loader, device, BATCH_SIZE)
 data_list += predict_data(model_pri, pri_loader, device, BATCH_SIZE)
 pred_df = pd.DataFrame(data_list, columns=["id_seqpos"] + target_cols)
 print(pred_df.head())
 print(pred_df.tail())
 pred_df_list.append(pred_df)
 c += 1
data_dic = dict(id_seqpos=pred_df_list[0]["id_seqpos"])
 for col in target_cols:
vals = np.zeros(pred_df_list[0][col].shape[0])
 for df in pred_df_list:
vals += df[col].values
data_dic[col] = vals / float(c)
 pred_df_avg = pd.DataFrame(data_dic, columns=["id_seqpos"] + target_cols)
 print(pred_df_avg.head())
 pred_df_avg.to_csv("./submission.csv", index=False)
<init_hyperparams>
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 25
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [40, 15, 5, 5, 5, 8]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline<load_from_csv>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<concatenate>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape<categorify>
predictors = ['Sex', 'Age', 'Pclass', 'Fare', 'Parch', 'SibSp', 'FamilySize', 'Title']
target = 'Survived'
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )<data_type_conversions>
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
As.shape, As_pub.shape, As_pri.shape<categorify>
clf.fit(train_X, train_Y )
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )<compute_test_metric>
preds = clf.predict(valid_X )
def mcrmse(t, p, seq_len_target = seq_len_target):
score = np.mean(np.sqrt(np.mean(( p - y_va)** 2, axis = 2)) [:, :seq_len_target])
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(( t - y)** 2, axis = 2)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 128, rate = 0.4)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam<train_model>
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 ae_model.fit([X_node, As], [X_node[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 ae_model.fit([X_node_pub, As_pub], [X_node_pub[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 ae_model.fit([X_node_pri, As_pri], [X_node_pri[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("./base_ae" )<split>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
kfold = KFold(5, shuffle = True, random_state = 42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As)) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )<predict_on_test>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
predictors = ['FamilySize', 'Title', 'Class*Age']
target = 'Survived'
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<set_options>
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()<compute_test_metric>
rf_clf = clf.fit(train_X, train_Y )
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True)
 def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test)
 kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_<choose_model_class>
preds = clf.predict(valid_X )
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100, hidden_dim=256, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 return model
keras_model = build_model()
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv0.h5')
 keras_model.layers<set_options>
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )
device = torch.device('cuda:%s'%0 if torch.cuda.is_available() else 'cpu')
 def Init_params(shape,w=None,b=None):
if w is None:
w = torch.nn.Parameter(torch.empty(*shape))
nn.init.xavier_uniform_(w)
 else:
w = torch.nn.Parameter(w)
 if b is None:
b = torch.nn.Parameter(torch.zeros(shape[1]))
else:
b = torch.nn.Parameter(b)
 return w,b
class GRU(nn.Module):
def __init__(self,input_dim,hidden_dim,w_i=None,b_i=None,w_h=None,b_h=None):
super(GRU, self ).__init__()
self.w_i,self.b_i = Init_params([input_dim,3*hidden_dim],w_i,b_i)
 self.w_h,self.b_h = Init_params([hidden_dim,3*hidden_dim],w_h,b_h)
 self.hd = hidden_dim
def forward(self,x):
hidden = torch.zeros(( x.shape[0], self.hd)).to(device)
 output = []
for i in range(x.shape[1]):
x_z = torch.matmul(x[:,i,:],self.w_i[:,:self.hd])+ self.b_i[:self.hd]
x_r = torch.matmul(x[:,i,:],self.w_i[:,self.hd:2*self.hd])+ self.b_i[self.hd:2*self.hd]
x_n = torch.matmul(x[:,i,:],self.w_i[:,2*self.hd:])+ self.b_i[2*self.hd:]
h_z = torch.matmul(hidden,self.w_h[:,:self.hd])+ self.b_h[:self.hd]
h_r = torch.matmul(hidden,self.w_h[:,self.hd:2*self.hd])+ self.b_h[self.hd:2*self.hd]
h_n = torch.matmul(hidden,self.w_h[:,2*self.hd:])+ self.b_h[2*self.hd:]
z = torch.sigmoid(x_z+h_z)
 r = torch.sigmoid(x_r+h_r)
 n = torch.tanh(x_n+r*h_n)
 h =(1-z)*n + z*hidden
hidden = h
output.append(h.unsqueeze(1))
return torch.cat(output,1)
 class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
num_target=5
w = torch.Tensor(keras_model.layers[2].get_weights() [0])
 self.cate_emb = nn.Embedding.from_pretrained(w,freeze=False)
 self.gru = GRU(100*3+3, 256, torch.Tensor(keras_model.layers[-4].get_weights() [0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [2][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [1] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [2][1] ).contiguous())
 self.reverse_gru = GRU(100*3+3, 256, torch.Tensor(keras_model.layers[-4].get_weights() [3] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [5][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [4] ).contiguous() ,
torch.Tensor(keras_model.layers[-4].get_weights() [5][1] ).contiguous())
 self.gru1 = GRU(512, 256, torch.Tensor(keras_model.layers[-3].get_weights() [0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [2][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [1] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [2][1] ).contiguous())
 self.reverse_gru1 = GRU(512, 256, torch.Tensor(keras_model.layers[-3].get_weights() [3] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [5][0] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [4] ).contiguous() ,
torch.Tensor(keras_model.layers[-3].get_weights() [5][1] ).contiguous())
 self.predict = nn.Linear(512,num_target)
 for i,(n,p)in enumerate(self.predict.named_parameters()):
if i == 0:
p.data = torch.nn.Parameter(torch.Tensor(keras_model.layers[-1].get_weights() [0].T ).contiguous())
 if i == 1:
p.data = torch.nn.Parameter(torch.Tensor(keras_model.layers[-1].get_weights() [1] ).contiguous())
 def forward(self, cateX,contX):
cate_x = self.cate_emb(cateX ).view(cateX.shape[0],cateX.shape[1],-1)
 sequence = torch.cat([cate_x,contX],-1)
 x = self.gru(sequence)
 reverse_x = torch.flip(self.reverse_gru(torch.flip(sequence,[1])) ,[1])
 sequence = torch.cat([x,reverse_x],-1)
 x = self.gru1(sequence)
 reverse_x = torch.flip(self.reverse_gru1(torch.flip(sequence,[1])) ,[1])
 x = torch.cat([x,reverse_x],-1)
 x = F.dropout(x,0.5,training=self.training)
 predict = self.predict(x)
 return predict
pytorch_model = Net()
pytorch_model.to(device )<prepare_x_and_y>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
x = preprocess_inputs(train[:1])
 cate_x = torch.LongTensor(x[:,:,:3] ).to(device)
 cont_x = torch.Tensor(x[:,:,3:] ).to(device)
 y = np.array(train[:1][pred_cols].values.tolist() ).transpose(( 0, 2, 1))<predict_on_test>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
keras_y = keras_model.predict(x)
 keras_y<predict_on_test>
test_X = test_df[predictors]
pred_Y = clf.predict(test_X )
pytorch_model.eval()
pytorch_y = pytorch_model(cate_x,cont_x ).detach().cpu().numpy()
pytorch_y<compute_test_metric>
submission = pd.DataFrame({"PassengerId": test_df["PassengerId"],"Survived": pred_Y})
 submission.to_csv('submission.csv', index=False )
np.mean(np.abs(keras_y-pytorch_y[:,:68,:]))<load_pretrained>
rf_clf = clf.fit(train_X, train_Y )
gkf = GroupKFold(n_splits=5)
 keras_predict = []
pytorch_predict = []
targets = []
for fold,(train_index, valid_index)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv%s.h5'%fold)
 t_valid = train.iloc[valid_index]
t_valid = t_valid[t_valid['SN_filter'] == 1]
valid_x = preprocess_inputs(t_valid)
 valid_count = valid_x.shape[0]
valid_cate_x = torch.LongTensor(valid_x[:,:,:3])
 valid_cont_x = torch.Tensor(valid_x[:,:,3:])
 valid_y = torch.Tensor(np.array(t_valid[pred_cols].values.tolist() ).transpose(( 0, 2, 1)))
 valid_data = TensorDataset(valid_cate_x,valid_cont_x,valid_y)
 valid_data_loader = DataLoader(dataset=valid_data,shuffle=False,batch_size=32,num_workers=1)
 valid_y = valid_y.numpy()
targets.append(valid_y)
 keras_predict.append(keras_model.predict(valid_x))
pytorch_model = Net()
pytorch_model.to(device)
 pytorch_model.eval()
all_pred = []
for data in valid_data_loader:
cate_x,cont_x,y = [x.to(device)for x in data]
outputs = pytorch_model(cate_x,cont_x)
 all_pred.append(outputs.detach().cpu().numpy())
 all_pred = np.concatenate(all_pred,0)[:,:68,:]
pytorch_predict.append(all_pred )<compute_test_metric>
parameters = {
'n_estimators':(50, 75,100),
'max_features':('auto', 'sqrt'),
'max_depth':(3,4,5),
'min_samples_split':(2,5,10),
'min_samples_leaf':(1,2,3)
 }
for i in range(5):
print('fold %s output difference between Keras and Pytorch:'%i,np.mean(np.abs(keras_predict[i]-pytorch_predict[i])) )<compute_test_metric>
%%time
gs_clf = GridSearchCV(rf_clf, parameters, n_jobs=-1, cv = 5, verbose = 5)
 gs_clf = gs_clf.fit(train_X, train_Y )
def Metric(target,pred):
metric = 0
for i in range(target.shape[-1]):
metric +=(np.sqrt(np.mean(( target[:,:,i]-pred[:,:,i])**2)) /target.shape[-1])
 return metric<compute_test_metric>
print('Best scores:',gs_clf.best_score_)
 print('Best params:',gs_clf.best_params_ )
for i in range(5):
print('fold %s'%i,'|','metric of keras outputs:%.6f'%Metric(targets[i],keras_predict[i]),'|','metric of pytorch outputs:%.6f'%Metric(targets[i],pytorch_predict[i]))<compute_test_metric>
preds = gs_clf.predict(valid_X )
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=5):
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
for i in range(5):
print('fold %s'%i,mcrmse(targets[i],keras_predict[i]))<compute_test_metric>
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )
for i in range(5):
print('fold %s'%i,K.mean(mcrmse(targets[i],keras_predict[i])) )<create_dataframe>
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))
def Pred(df):
test_x = preprocess_inputs(df)
 test_cate_x = torch.LongTensor(test_x[:,:,:3])
 test_cont_x = torch.Tensor(test_x[:,:,3:])
 test_data = TensorDataset(test_cate_x,test_cont_x)
 test_data_loader = DataLoader(dataset=test_data,shuffle=False,batch_size=64,num_workers=1)
 all_id = []
for i,row in df.iterrows() :
for j in range(row['seq_length']):
all_id.append(row['id']+'_%s'%j)
 all_id = np.array(all_id ).reshape(-1,1)
 all_pred = np.zeros(len(all_id)*5 ).reshape(len(all_id),5)
 for fold in range(5):
keras_model.load_weights('.. /input/gru-lstm-with-feature-engineering-and-augmentation/modelGRU_LSTM1_cv%s.h5'%fold)
 model = Net()
model.to(device)
 model.eval()
t_all_pred = []
for data in test_data_loader:
cate_x,cont_x = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 t_all_pred.append(outputs.detach().cpu().numpy())
 t_all_pred = np.concatenate(t_all_pred,0)
 all_pred += t_all_pred.reshape(-1,5)
 all_pred /= 5
sub = pd.DataFrame(all_pred,columns=['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C'])
 sub['id_seqpos'] = all_id
return sub
public_sub = Pred(test.loc[test['seq_length']==107])
 private_sub = Pred(test.loc[test['seq_length']==130])
 pytorch_sub = pd.concat([public_sub,private_sub] ).reset_index(drop=True)
 pytorch_sub = pytorch_sub[['id_seqpos']+pred_cols]<sort_values>
test_X = test_df[predictors]
pred_Y = gs_clf.predict(test_X )
pytorch_sub = pytorch_sub.sort_values(by=['id_seqpos'] ).reset_index(drop=True )<load_from_csv>
submission = pd.DataFrame({"PassengerId": test_df["PassengerId"],"Survived": pred_Y})
 submission.to_csv('submission_hyperparam_optimization.csv', index=False )
keras_sub = pd.read_csv('.. /input/gru-lstm-with-feature-engineering-and-augmentation/submission.csv' )<sort_values>
NUMBER_KFOLDS = 5
kf = KFold(n_splits = NUMBER_KFOLDS, random_state = RANDOM_STATE, shuffle = True )
keras_sub = keras_sub.sort_values(by=['id_seqpos'] ).reset_index(drop=True )<compute_test_metric>
class SklearnBasicClassifier(object):
def __init__(self, clf, seed=2018, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
 def get_feature_importances(self,x,y):
return(self.clf.fit(x,y ).feature_importances_)
np.mean(np.abs(keras_sub[pred_cols].values-pytorch_sub[pred_cols].values))<save_to_csv>
ntrain = train_df.shape[0]
ntest = test_df.shape[0]
def get_oof_predictions(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NUMBER_KFOLDS, ntest))
for i,(train_idx, valid_idx)in enumerate(kf.split(train_df)) :
clf.train(x_train[train_idx], y_train[train_idx])
 oof_train[valid_idx] = clf.predict(x_train[valid_idx])
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
pytorch_sub.to_csv('./submission.csv',index=False )<set_options>
ada_params = {
'n_estimators': 200,
'learning_rate' : 0.75
}
cat_params = {
'iterations': 150,
'learning_rate': 0.02,
'depth': 12,
'bagging_temperature':0.2,
'od_type':'Iter',
'metric_period':400,
}
ext_params = {
'n_jobs': -1,
'n_estimators':100,
'max_depth': 8,
'min_samples_leaf': 3,
'verbose': 0
}
gbm_params = {
'n_estimators': 200,
'max_depth': 5,
'min_samples_leaf': 3,
'verbose': 0
}
rfo_params = {
'n_jobs': -1,
'n_estimators': 50,
'max_depth': 5,
'min_samples_leaf': 5,
'max_features' : 'auto',
'verbose': 0
}
svc_params = {
'kernel' : 'linear',
'C' : 0.02
}
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()
Ver='GRU_LSTM48k'
aug_data = '.. /input/augmented-data-for-stanford-covid-vaccine/48k_augment.csv'
debug = False<choose_model_class>
ada = SklearnBasicClassifier(clf=AdaBoostClassifier, seed=RANDOM_STATE, params=ada_params)
 cat = SklearnBasicClassifier(clf=CatBoostClassifier, seed=RANDOM_STATE, params=cat_params)
 ext = SklearnBasicClassifier(clf=ExtraTreesClassifier, seed=RANDOM_STATE, params=ext_params)
 gbm = SklearnBasicClassifier(clf=GradientBoostingClassifier, seed=RANDOM_STATE, params=gbm_params)
 rfo = SklearnBasicClassifier(clf=RandomForestClassifier, seed=RANDOM_STATE, params=rfo_params)
 svc = SklearnBasicClassifier(clf=SVC, seed=RANDOM_STATE, params=svc_params )
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100, hidden_dim=256, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 return model<categorify>
predictors = ['FamilySize', 'Title', 'Class*Age']
target = 'Survived'
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score<load_from_disk>
y_train = train_df['Survived'].values
train = train_df[predictors]
test = test_df[predictors]
x_train = train.values
x_test = test.values
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True )<load_pretrained>
print("Start training")
 ada_oof_train, ada_oof_test = get_oof_predictions(ada, x_train, y_train, x_test)
 print("End AdaBoost")
 cat_oof_train, cat_oof_test = get_oof_predictions(cat, x_train, y_train, x_test)
 print("End CatBoost")
 ext_oof_train, ext_oof_test = get_oof_predictions(ext, x_train, y_train, x_test)
 print("End ExtraTrees")
 rfo_oof_train, rfo_oof_test = get_oof_predictions(rfo,x_train, y_train, x_test)
 print("End RandomForest")
 gbm_oof_train, gbm_oof_test = get_oof_predictions(gbm,x_train, y_train, x_test)
 print("End GradientBoost")
 svc_oof_train, svc_oof_test = get_oof_predictions(svc,x_train, y_train, x_test)
 print("End training" )
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )<train_model>
ada_feature_importance = ada.get_feature_importances(x_train,y_train)
 cat_feature_importance = cat.get_feature_importances(x_train,y_train)
 ext_feature_importance = ext.get_feature_importances(x_train,y_train)
 gbm_feature_importance = gbm.get_feature_importances(x_train,y_train)
 rfo_feature_importance = rfo.get_feature_importances(x_train,y_train )
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_<load_from_csv>
base_predictions_train = pd.DataFrame({
'AdaBoost': ada_oof_train.ravel() ,
'CatBoost': cat_oof_train.ravel() ,
'ExtraTrees': ext_oof_train.ravel() ,
'GradientBoost': gbm_oof_train.ravel() ,
'RandomForest': rfo_oof_train.ravel() ,
'SVM': svc_oof_train.ravel()
})
 base_predictions_train.head(10 )
aug_df = pd.read_csv(aug_data)
 display(aug_df.head() )<merge>
x_train = np.concatenate(( ada_oof_train, cat_oof_train, ext_oof_train, gbm_oof_train, rfo_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( ada_oof_test, cat_oof_test, ext_oof_test, gbm_oof_test, rfo_oof_test, svc_oof_test), axis=1 )
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )<split>
clf = xgb.XGBClassifier(
 learning_rate = 0.02,
n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 )
if debug:
train = train[:200]
test = test[:200]<split>
xgbm = clf.fit(x_train, y_train )
def train_and_predict(type = 0, FOLD_N = 5):
gkf = GroupKFold(n_splits=FOLD_N)
 public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn)
 y_trn = np.array(trn[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val)
 y_val = np.array(val[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
model = build_model(type=type)
 model_short = build_model(seq_len=107, pred_len=107,type=type)
 model_long = build_model(seq_len=130, pred_len=130,type=type)
 history = model.fit(
 x_trn, y_trn,
validation_data =(x_val, y_val),
batch_size=64,
epochs=60,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5')
 ] 
)
 fig = px.line(
 history.history, y=['loss', 'val_loss'],
labels={'index': 'epoch', 'value': 'Mean Squared Error'},
title='Training History')
 fig.show()
model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_short.load_weights(f'model{Ver}_cv{cv}.h5')
 model_long.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
public_preds = model_short.predict(public_inputs)/FOLD_N
private_preds = model_long.predict(private_inputs)/FOLD_N
else:
public_preds += model_short.predict(public_inputs)/FOLD_N
private_preds += model_long.predict(private_inputs)/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds<predict_on_test>
predictions = xgbm.predict(x_test )
<prepare_output><EOS> 
submissionStacking = pd.DataFrame({ 'PassengerId': test_df["PassengerId"],'Survived': predictions })
 submissionStacking.to_csv("submission_ensamble.csv", index=False )