Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<load_from_csv>
warnings.filterwarnings('ignore')
X = StandardScaler().fit(data ).transform(data)
 X.shape
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = StandardScaler().fit(data ).transform(test )<choose_model_class>
def seed_everything(seed = 34):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything()
def create_model(dropout_rate_l1=0.3 , dropout_rate_l2=0.3):
model = Sequential()
model.add(Dense(600, input_dim=192, init='uniform'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(dropout_rate_l1))
model.add(Dense(300, init='uniform'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(dropout_rate_l2))
model.add(Dense(99, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 return model<split>
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv('.. /input/stanford-covid-vaccine/sample_submission.csv' )
X_train,X_test,y_train,y_test = train_test_split(X, y_cat, test_size = 0.1, random_state = 7 )<train_model>
print(f"Samples with signal_to_noise greater than 1: {len(train.loc[(train['signal_to_noise'] > 1)])}")
 print(f"Samples with SN_filter = 1: {len(train.loc[(train['SN_filter'] == 1)])}")
 print(f"Samples with signal_to_noise greater than 1, but SN_filter == 0: {len(train.loc[(train['signal_to_noise'] > 1)&(train['SN_filter'] == 0)])}" )
model = create_model()
history_main = model.fit(X_train,y_train,batch_size=192, nb_epoch=400, verbose=2, validation_data=(X_test, y_test))<compute_test_metric>
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test)
 train.head()
model.evaluate(X, y_cat )<save_to_csv>
AUGMENT=True
yPred = model.predict_proba(test)
 col = or_data.species.unique()
col.sort()
yPred = pd.DataFrame(yPred, index=index, columns=col)
 yPred.to_csv('result_2.csv', index=True )<load_pretrained>
aug_df = pd.read_csv('.. /input/openvaccineaugmented/aug_data_n2.csv')
 print(aug_df.shape)
 aug_df.head()
!unzip /kaggle/input/quora-insincere-questions-classification/embeddings.zip -d /kaggle/input/embeddings
!ls -l /kaggle/input/embeddings<init_hyperparams>
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
max_len = 50
lower = True
trunc = 'pre'
max_features = 120000
n_vocab = max_features
clean_num = 0
n_models = 5
epochs = 8
batch_size = 512
drop_last = True
hidden_dim = 128
fix_embedding = True
unk_uni = True
n_embed = 2
embed_dim = n_embed * 300
proj_dim = hidden_dim
bidirectional = True
n_layers = 1
rnn_dim = hidden_dim
dense_dim = 2 * rnn_dim if bidirectional else rnn_dim
mu = 0.9
updates_per_epoch = 10
threshold = 0.353
test_batch_size = 8 * batch_size
test_batch_size = batch_size
def seed_torch(seed=1):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed)
 device = torch.device('cuda')if torch.cuda.is_available() else 'cpu'
def get_param_size(model, trainable=True):
if trainable:
psize = np.sum([np.prod(p.size())for p in model.parameters() if p.requires_grad])
 else:
psize = np.sum([np.prod(p.size())for p in model.parameters() ])
 return psize<init_hyperparams>
print(f"Samples in train before augmentation: {len(train)}")
 print(f"Samples in test before augmentation: {len(test)}")
 if AUGMENT:
train = aug_data(train)
 test = aug_data(test)
 print(f"Samples in train after augmentation: {len(train)}")
 print(f"Samples in test after augmentation: {len(test)}")
 print(f"Unique sequences in train: {len(train['sequence'].unique())}")
 print(f"Unique sequences in test: {len(test['sequence'].unique())}" )
class EMA() :
def __init__(self, model, mu, level='batch', n=1):

 self.mu = mu
self.level = level
self.n = n
self.cnt = self.n
self.shadow = {}
for name, param in model.named_parameters() :
if param.requires_grad:
self.shadow[name] = param.data
def _update(self, model):
for name, param in model.named_parameters() :
if param.requires_grad:
new_average =(1 - self.mu)* param.data + self.mu * self.shadow[name]
self.shadow[name] = new_average.clone()
def set_weights(self, ema_model):
for name, param in ema_model.named_parameters() :
if param.requires_grad:
param.data = self.shadow[name]
def on_batch_end(self, model):
if self.level is 'batch':
self.cnt -= 1
if self.cnt == 0:
self._update(model)
 self.cnt = self.n
def on_epoch_end(self, model):
if self.level is 'epoch':
self._update(model)
<import_modules>
DENOISE = False
class GlobalMaxPooling1D(nn.Module):
def __init__(self):
super(GlobalMaxPooling1D, self ).__init__()
def forward(self, inputs):
z, _ = torch.max(inputs, 1)
 return z
def __repr__(self):
return self.__class__.__name__ + '() '
class GRUModel(nn.Module):
def __init__(self, n_vocab, embed_dim, proj_dim, rnn_dim, n_layers, bidirectional, dense_dim,
padding_idx=0, pretrained_embedding=None, fix_embedding=True,
n_out=1):
super(GRUModel, self ).__init__()
self.n_vocab = n_vocab
self.embed_dim = embed_dim
self.n_layers = n_layers
self.dense_dim = dense_dim
self.n_out = n_out
self.bidirectional = bidirectional
self.fix_embedding = fix_embedding
self.padding_idx = padding_idx
if pretrained_embedding is not None:
self.embed = nn.Embedding.from_pretrained(pretrained_embedding, freeze=fix_embedding)
 self.embed.padding_idx = self.padding_idx
else:
self.embed = nn.Embedding(self.n_vocab, self.embed_dim, padding_idx=self.padding_idx)
 self.proj = nn.Linear(embed_dim, proj_dim)
 self.proj_act = nn.ReLU()
self.gru = nn.GRU(proj_dim, rnn_dim, self.n_layers,
batch_first=True, bidirectional=bidirectional)
 self.pooling = GlobalMaxPooling1D()
in_dim = 2 * rnn_dim if self.bidirectional else rnn_dim
self.dense = nn.Linear(in_dim, dense_dim)
 self.dense_act = nn.ReLU()
self.out_linear = nn.Linear(dense_dim, n_out)
 self.init_weights()
def init_weights(self):
for name, param in self.named_parameters() :
if name.find('embed')> -1:
continue
elif name.find('weight')> -1 and len(param.size())> 1:
nn.init.xavier_uniform_(param)
 def forward(self, inputs):
x = self.embed(inputs)
 x = self.proj_act(self.proj(x))
x, hidden = self.gru(x)
 x = self.pooling(x)
 x = self.dense_act(self.dense(x))
x = self.out_linear(x)
 return x
def fit(self, dataloader, epochs, optimizer, callbacks=None):
for i in range(epochs):
run_epoch(model, dataloader, optimizer, callbacks=callbacks)
 def predict(self, dataloader):
preds = []
with torch.no_grad() :
for batch in dataloader:
batch = tuple(t.to(device)for t in batch)
 X_batch = batch[0]
preds.append(self.forward(X_batch ).data.cpu())
 return torch.cat(preds)
 def predict_proba(self, dataloader):
return torch.sigmoid(self.predict(dataloader)).data.numpy()<categorify>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
class TextClassificationDataset(Dataset):

 def __init__(self, token_ids_s, labels=None, max_len=1000, training=True, sort=True):
self.training = training
if labels is None:
self.labels = torch.ones(len(token_ids_s), dtype=torch.long)
 else:
self.labels = torch.LongTensor(labels)
 seq_lens = []
self.inputs = []
for e, token_ids in enumerate(token_ids_s):
seq_lens.append(len(token_ids))
input_ids = torch.LongTensor(token_ids[:max_len])
 self.inputs.append(input_ids)
 if self.training and sort:
self.indices = np.argsort(seq_lens)
 self.inputs = [self.inputs[i] for i in self.indices]
self.labels = self.labels[self.indices]
def __len__(self):
return len(self.inputs)
 def __getitem__(self, idx):
return self.inputs[idx], self.labels[idx]
def set_labels(self, labels):
self.labels = torch.tensor(labels, dtype=torch.long)
 class BatchIterator(object):
def __init__(self, dataset, collate_fn, batch_size,
shuffle=True, drop_last=True):
self.dataset = dataset
self.collate_fn = collate_fn
self.batch_size = batch_size
self.size = len(dataset)
 self.shuffle = shuffle
if drop_last:
self.num_batches = self.size // batch_size
else:
self.num_batches =(self.size + self.batch_size - 1)// self.batch_size
def __iter__(self):
if self.shuffle:
indices = np.random.choice(self.num_batches, self.num_batches, replace=False)
 else:
indices = range(self.num_batches)
 for idx in indices:
left = self.batch_size * idx
yield(self.collate_fn(self.dataset[left: left + self.batch_size]))
def __len__(self):
return self.num_batches
def collate_fn(batch):
xy_batch = [pad_sequence(batch[0], batch_first=True), batch[1]]
return xy_batch
def run_epoch(model, dataloader, optimizer, callbacks=None,
criterion=nn.BCEWithLogitsLoss() , verbose_step=10000):
t1 = time.time()
tr_loss = 0
for step, batch in enumerate(dataloader):
batch = tuple(t.to(device)for t in batch)
 x_batch, y_batch = batch
model.zero_grad()
outputs = model(x_batch)
 loss = criterion(outputs[:, 0], y_batch.float())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
if callbacks is not None:
for func in callbacks:
func.on_batch_end(model)
 if(step + 1)% verbose_step == 0:
loss_now = tr_loss /(step + 1)
 print(f'step:{step+1} loss:{loss_now:.7f} time:{time.time() - t1:.1f}s')
 if callbacks is not None:
for func in callbacks:
func.on_epoch_end(model)
 return tr_loss /(step + 1 )<compute_train_metric>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def load_glove(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))
all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns glove', len(unknown_words))
print(unknown_words[-10:])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_wiki(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE)if len(o)> 100)
 all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns wiki', len(unknown_words))
print(unknown_words[-10:])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_parag(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(' '))
for o in open(EMBEDDING_FILE, encoding='utf8', errors='ignore')
 if len(o)> 100)
 all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns parag', len(unknown_words))
print(unknown_words[-10:])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_ggle(word_index, max_features, unk_uni):
EMBEDDING_FILE = data_path + 'embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 embed_size = embeddings_index.get_vector('known' ).size
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix[i] = embedding_vector
else:
word_lower = word.lower()
if word_lower in embeddings_index:
embedding_matrix[i] = embeddings_index.get_vector(word_lower)
 else:
unknown_words.append(( word, i))
print('
Total unknowns ggle', len(unknown_words))
print(unknown_words[-10:])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_all_embeddings(tokenizer, max_features, clean_num=False, unk_uni=True):
word_index = tokenizer.word_index
if clean_num == 2:
ggle_word_index = {}
for word, i in word_index.items() :
ggle_word_index[clean_numbers(word)] = i
else:
ggle_word_index = word_index
embedding_matrix_1, u1 = load_glove(word_index, max_features, unk_uni)
 embedding_matrix_2, u2 = load_wiki(word_index, max_features, unk_uni)
 embedding_matrix_3, u3 = load_parag(word_index, max_features, unk_uni)
 embedding_matrix_4, u4 = load_ggle(ggle_word_index, max_features, unk_uni)
 embedding_matrix = np.concatenate(( embedding_matrix_1,
embedding_matrix_2,
embedding_matrix_3,
embedding_matrix_4), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4
gc.collect()
return embedding_matrix
def setup_emb(tr_X, max_features=50000, clean_num=2, unk_uni=True):
tokenizer = Tokenizer(num_words=max_features, lower=True, filters='')
 tokenizer.fit_on_texts(tr_X)
 print(tokenizer)
 print('len(vocab)', len(tokenizer.word_index))
embedding_matrix = load_all_embeddings(tokenizer, max_features=max_features,
clean_num=clean_num, unk_uni=unk_uni)
 return tokenizer, embedding_matrix 
<categorify>
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea], 2 )
puncts = ',.": )(-!?|;'$&/[]>%=
―¥▓—‹─▒：¼⊕▼▪†■’▀¨▄♫☆é¯♦¤▲è¸¾Ã⋅‘∞∙）↓、│（»，♪╩╚³・╦╣╔╗▬❤ïØ¹≤‡√'
def clean_text(x, puncts=puncts):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def prepare_data(train_df, test_df, max_len, max_features, trunc='pre',
lower=False, clean_num=2, unk_uni=True):
train_df = train_df.copy()
train_y = train_df['target'].values
if lower:
train_df['question_text'] = train_df['question_text'].apply(lambda x: x.lower())
 test_df['question_text'] = test_df['question_text'].apply(lambda x: x.lower())
 train_df['question_text'] = train_df['question_text'].apply(
 lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].apply(
 lambda x: clean_text(x))
if clean_num == 1:
train_df['question_text'] = train_df['question_text'].apply(
 lambda x: clean_numbers(x))
test_df['question_text'] = test_df['question_text'].apply(
 lambda x: clean_numbers(x))
train_df['question_text'] = train_df['question_text'].fillna('_
test_df['question_text'] = test_df['question_text'].fillna('_
train_X = train_df['question_text'].values
test_X = test_df['question_text'].values
tokenizer, embedding_matrix = setup_emb(train_X,
max_features=max_features,
clean_num=clean_num, unk_uni=unk_uni)
 train_X_ids = tokenizer.texts_to_sequences(train_X)
 train_dataset = TextClassificationDataset(train_X_ids, train_y, sort=False)
 train_loader = BatchIterator(train_dataset, collate_fn, batch_size)
 test_X_ids = tokenizer.texts_to_sequences(test_X)
 test_dataset = TextClassificationDataset(test_X_ids, training=False)
 test_loader = BatchIterator(test_dataset, collate_fn, batch_size, shuffle=False, drop_last=False)
 embedding_matrix = torch.Tensor(embedding_matrix)
 return train_loader, test_loader, embedding_matrix<load_from_csv>
if DENOISE:
train = train[train['signal_to_noise'] >.25]
ids_s = [list(range(300)) , list(range(300, 600)) ,
list(range(600, 900)) , list(range(900, 1200)) ]
cols_s = [ids_s[0] + ids_s[1],
ids_s[0] + ids_s[2],
ids_s[1] + ids_s[2],
ids_s[1] + ids_s[3],
ids_s[2] + ids_s[3]]
train_df = pd.read_csv(data_path + 'quora-insincere-questions-classification/' + 'train.csv')
 test_df = pd.read_csv(data_path + 'quora-insincere-questions-classification/' + 'test.csv')
 display(test_df.head())
 print('Train : ', train_df.shape)
 print('Test : ', test_df.shape)
<choose_model_class>
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(target_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
ema_n = int(train_df.shape[0] /(updates_per_epoch * batch_size))
test_pr = np.zeros(( len(test_df), 1))
for i in range(n_models):
cols_in_use = cols_s[i % len(cols_s)]
model = GRUModel(n_vocab, embed_dim, proj_dim, rnn_dim, n_layers, bidirectional, dense_dim,
pretrained_embedding=embedding_matrix[:, cols_in_use],
fix_embedding=fix_embedding, padding_idx=0)
 if i == 0:
print(model)
 print('
model.to(device)
 ema_model = copy.deepcopy(model)
 ema_model.eval()
optimizer = Adam([p for n, p in model.named_parameters() if p.requires_grad is True])
 ema = EMA(model, mu, n=ema_n)
 t2 = time.time()
model.train()
model.fit(train_loader, epochs, optimizer, callbacks=[ema])
 print(f'n_model:{i + 1} {(time.time() - t2)/ epochs:.1f}s/epoch')
 ema.set_weights(ema_model)
 ema_model.gru.flatten_parameters()
t3 = time.time()
test_pr += ema_model.predict_proba(test_loader)
 print(f'{time.time() - t3:.1f}s')
 test_pr /= n_models
test_pr =(test_pr > threshold ).astype(int)
 out_df = pd.DataFrame({"qid": test_df["qid"].values})
 out_df['prediction'] = test_pr
out_df.to_csv("submission.csv", index=False)
 print(f'Done:{time.time() - t0:.1f}s' )<import_modules>
def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def build_model(rnn='gru', convolve=False, conv_dim=512,
dropout=.4, sp_dropout=.2, embed_dim=200,
hidden_dim=256, layers=3,
seq_len=107, pred_len=68):
inputs = tf.keras.layers.Input(shape=(seq_len, 5))
categorical_feats = inputs[:, :, :3]
numerical_feats = inputs[:, :, 3:]
embed = tf.keras.layers.Embedding(input_dim=len(token2int),
output_dim=embed_dim )(categorical_feats)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.concatenate([reshaped, numerical_feats], axis=2)
 hidden = tf.keras.layers.SpatialDropout1D(sp_dropout )(reshaped)
 if convolve:
hidden = tf.keras.layers.Conv1D(conv_dim, 5, padding='same', activation=tf.keras.activations.swish )(hidden)
 if rnn is 'gru':
for _ in range(layers):
hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif rnn is 'lstm':
for _ in range(layers):
hidden = lstm_layer(hidden_dim, dropout )(hidden)
 out = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(out)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer=adam, loss=mcrmse)
 return model
import pandas as pd
from tqdm import tqdm
import re
import numpy as np
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from gensim.models import KeyedVectors
from sklearn.preprocessing import StandardScaler
import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense,Input,Embedding,Dropout,Reshape,Flatten,LSTM,Bidirectional
from keras.layers.merge import concatenate
from keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping,ModelCheckpoint<string_transform>
def train_and_infer(rnn, STRATIFY=True, FOLDS=4, EPOCHS=50, BATCH_SIZE=64,
REPEATS=3, SEED=34, VERBOSE=2):
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
private_preds = np.zeros(( private_df.shape[0], 130, 5))
public_preds = np.zeros(( public_df.shape[0], 107, 5))
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
histories = []
gkf = GroupKFold(n_splits=FOLDS)
 kf=KFold(n_splits=FOLDS, random_state=SEED)
 kmeans_model = KMeans(n_clusters=200, random_state=SEED ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
for _ in range(REPEATS):
for f,(train_index, val_index)in enumerate(( gkf if STRATIFY else kf ).split(train,
train['reactivity'], train['cluster_id'] if STRATIFY else None)) :
lr_callback = tf.keras.callbacks.ReduceLROnPlateau(patience=8,
factor=.1,
verbose=VERBOSE)
 save = tf.keras.callbacks.ModelCheckpoint(f'model-{f}.h5')
 epsilon =.1
sample_weighting = np.log1p(train.iloc[train_index]['signal_to_noise'] + epsilon)/ 2
trn = train.iloc[train_index]
trn_ = preprocess_inputs(trn)
 trn_labs = np.array(trn[target_cols].values.tolist() ).transpose(( 0, 2, 1))
val = train.iloc[val_index]
val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
val_ = preprocess_inputs(val)
 val_labs = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
model = build_model(rnn=rnn)
 model_short = build_model(rnn=rnn,seq_len=107, pred_len=107)
 model_long = build_model(rnn=rnn,seq_len=130, pred_len=130)
 history = model.fit(
 trn_, trn_labs,
validation_data =(val_, val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
sample_weight=sample_weighting,
callbacks=[save, lr_callback],
verbose=VERBOSE 
)
 histories.append(history)
 model.load_weights(f'model-{f}.h5')
 model_short.load_weights(f'model-{f}.h5')
 model_long.load_weights(f'model-{f}.h5')
 holdouts.append(train.iloc[val_index])
 holdout_preds.append(model.predict(val_all))
public_preds += model_short.predict(public_inputs)/(FOLDS * REPEATS)
 private_preds += model_long.predict(private_inputs)/(FOLDS * REPEATS)
 del model, model_short, model_long
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds, histories
def text_to_wordlist(text , remove_stopwords = False , stem_words = False):
text = text.lower().split()
if remove_stopwords:
stops = set(stopwords.words('english'))
text= [w for w in text if not w in stops]
text = ' '.join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = ' '.join(stemmed_words)
 return(text )<load_from_csv>
gru_holdouts, gru_holdout_preds, public_df, gru_public_preds, private_df, gru_private_preds, gru_histories = train_and_infer(rnn='gru' )
def read_train_data(file):
texts = []
labels = []
df_train = pd.read_csv(file)
 line_num = 0
for idx in range(len(df_train)) :
texts.append(text_to_wordlist(df_train['question_text'][idx]))
labels.append(df_train['target'][idx])
 line_num += 1
return texts,labels<load_from_csv>
lstm_holdouts, lstm_holdout_preds, public_df, lstm_public_preds, private_df, lstm_private_preds, lstm_histories = train_and_infer(rnn='lstm' )
def read_test_data(file):
texts = []
ids = []
df_test = pd.read_csv(file)
 line_num = 0
for idx in range(len(df_test)) :
texts.append(text_to_wordlist(df_test['question_text'][idx]))
ids.append(df_test['qid'][idx])
 line_num += 1
return texts , ids<load_from_csv>
def format_predictions(test_df, test_preds, val=False):
preds = []
for df, preds_ in zip(test_df, test_preds):
for i, uid in enumerate(df['id']):
single_pred = preds_[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
if val: single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds.append(single_df)
 return pd.concat(preds ).groupby('id_seqpos' ).mean().reset_index() if AUGMENT else pd.concat(preds )
def preprocess_data(train_data_file , test_data_file , max_seq_len , split_ratio):
texts , labels = read_train_data(train_data_file)
 print('Finished loading train.csv: %s samples '%len(texts))
test_texts , test_ids = read_test_data(test_data_file)
 print('Finished loading test.csv: %s samples '%len(test_texts))
tokenizer = Tokenizer(num_words = 200000)
 tokenizer.fit_on_texts(texts + test_texts)
 word_index = tokenizer.word_index
print('%s tokens in total' % len(word_index))
train_sequences = tokenizer.texts_to_sequences(texts)
 test_sequence = tokenizer.texts_to_sequences(test_texts)
 x = pad_sequences(train_sequences , maxlen = max_seq_len , padding='post',truncating='post')
 test_x = pad_sequences(test_sequence , maxlen=max_seq_len , padding='post',truncating = 'post')
 num_samples = len(x)
 perm = np.random.permutation(num_samples)
 idx = int(num_samples * split_ratio)
 idx_train = perm[:idx]
idx_val = perm[idx:]
train_x = x[idx_train]
val_x = x[idx_val]
y = np.array(labels)
 train_y = y[idx_train]
val_y = y[idx_val]
print('shape of taining data: {}'.format(train_x.shape))
print('shape of training label: {}'.format(train_y.shape))
print('shape of val data: {}'.format(val_x.shape))
return train_x , train_y,val_x , val_y , test_x , test_ids , word_index 
<categorify>
def get_error(preds):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(preds, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))
print('' )
def load_embeddings_index(file , embedding_dim):
embeddings_index = {}
f = open(file)
 for line in tqdm(f):
values = line.split(' ')
 word = values[0]
coefs = np.asarray(values[1:],dtype = 'float32')
 embeddings_index[word] = coefs
print('Found %s word vectors' % len(embeddings_index)) 
<categorify>
gru_preds = [gru_public_preds, gru_private_preds]
lstm_preds = [gru_public_preds, gru_private_preds]
test_df = [public_df, private_df]
gru_preds = format_predictions(test_df, gru_preds)
 lstm_preds = format_predictions(test_df, lstm_preds )
def generate_embedding_layer(word_index , embeddings_index):
nb_words = len(word_index)+ 1
embeddings_matrix = np.zeros(( nb_words , embedding_dim))
word_out_netword = []
for word , i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
else:
word_out_netword.append(word)
 percent = round(100 * len(word_out_network)/ len(word_index),1)
 print('%s precent of words out of network' % percent)
 return embeddings_matrix<choose_model_class>
gru_weight =.5
lstm_weight =.5
def build_model(max_seq_len , word_index , embedding_dim , embedding_matrix):
inp = Input(shape=(max_seq_len,),dtype = 'int32')
 x = Embedding(len(word_index)+ 1,
embedding_dim,
input_length= max_seq_len,
weights = [embedding_matrix],
trainable = False )(inp)
 x = LSTM(64 , dropout = 0.2 , recurrent_dropout=0.2 )(x)
 x = Dropout(0,2 )(x)
 x = BatchNormalization()(x)
 x = Dense(32 , activation='relu' )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 preds = Dense(1,activation='sigmoid' )(x)
 model = Model(inputs = inp , outputs = preds)
 return model 
<load_from_csv>
blended_preds = pd.DataFrame()
blended_preds['id_seqpos'] = gru_preds['id_seqpos']
blended_preds['reactivity'] = gru_weight*gru_preds['reactivity'] + lstm_weight*lstm_preds['reactivity']
blended_preds['deg_Mg_pH10'] = gru_weight*gru_preds['deg_Mg_pH10'] + lstm_weight*lstm_preds['deg_Mg_pH10']
blended_preds['deg_pH10'] = gru_weight*gru_preds['deg_pH10'] + lstm_weight*lstm_preds['deg_pH10']
blended_preds['deg_Mg_50C'] = gru_weight*gru_preds['deg_Mg_50C'] + lstm_weight*lstm_preds['deg_Mg_50C']
blended_preds['deg_50C'] = gru_weight*gru_preds['deg_50C'] + lstm_weight*lstm_preds['deg_50C']
max_seq_len = 30
split_ratio = 0.8
train_file = '.. /input/quora-insincere-questions-classification/train.csv'
test_file = '.. /input/quora-insincere-questions-classification/test.csv'
train_x , train_y , val_x , val_y , test_x , test_idx , word_index = \
preprocess_data(train_file , test_file , max_seq_len , split_ratio )<categorify>
submission = sample_sub[['id_seqpos']].merge(blended_preds, on=['id_seqpos'])
 submission.head()
embedding_dim = 300
embedding_matrix = np.zeros(( max(list(word_index.values())) + 1 , embedding_dim),dtype = 'float')
 embedding_file = '.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'
f = open(embedding_file)
 for line in tqdm(f):
values = line.split(' ')
 word = values[0]
if word not in word_index:
continue
embedding_matrix[word_index[word]] = np.asarray(values[1:],dtype = 'float32')
 f.close<choose_model_class>
submission.to_csv(f'submission_new.csv', index=False)
 print('Submission saved' )