Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<train_model>
!pip install -r /kaggle/input/python-library/wheelhouse/requirements.txt --no-index --find-links /kaggle/input/python-library/wheelhouse
y = train_data.target
X = train_data.drop(['qid', 'target'], axis=1)
 test_data = test_data.drop(['qid'], axis= 1)
 print(X.shape)
 print(y.shape)
 print(test_data.shape)
<set_options>
print(np.__version__)
 print(pd.__version__)
 print(torch.__version__)
 print(torch_optimizer.__version__)
 print(iterstrat.__version__ )
del X; gc.collect()<load_pretrained>
SEED = 42
SEEDS = [4, SEED, 6, 89]
def set_seed(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 set_seed(SEED)
 N_EPOCHS = 250
N_FOLDS = 5
BATCH_SIZE = 128
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device
archive = zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')
<categorify>
X = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 y = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv' )
def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )<feature_engineering>
df_submission = pd.DataFrame(columns=y.columns)
 df_submission['sig_id'] = X_test['sig_id']
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = {}
for o in tqdm(archive.open('glove.840B.300d/glove.840B.300d.txt', 'r')) :
o = o.decode("utf-8")
 key , value = get_coefs(*o.split(" "))
embeddings_index.update({key:value})
 words = list(embeddings_index.keys())
 w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank<feature_engineering>
del X['sig_id']
del y['sig_id']
del X_test['sig_id']
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
embedding_matrix_1[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect()<feature_engineering>
cat_feature = ['cp_type', 'cp_dose']
for cat in cat_feature:
val_list = X[cat].unique()
for val in val_list:
X[f'{cat}_{val}'] = X[cat].apply(lambda i:i == val)
 X_test[f'{cat}_{val}'] = X_test[cat].apply(lambda i:i == val)
 del X[cat]
del X_test[cat]
print(X.columns[1:773])
 print(X.columns[773:873] )
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
embedding_matrix_2[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect()
del data
gc.collect()<feature_engineering>

 X = torch.tensor(X.astype(np.float32 ).to_numpy())
 y = torch.tensor(y.astype(np.float32 ).to_numpy())
 X_test = torch.tensor(X_test.astype(np.float32 ).to_numpy() )
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
embedding_matrix_3[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect() 
<drop_column>
class MoADataset(torch.utils.data.Dataset):
def __init__(self, X, y=None):
if y is not None:
assert X.shape[0] == y.shape[0]
self.X = X
self.y = y
def __len__(self):
return self.X.shape[0]
def __getitem__(self, idx):
if self.y is not None:
return self.X[idx], self.y[idx]
else:
return self.X[idx]
embedding_matrix = np.concatenate(( embedding_matrix_1 , embedding_matrix_2, embedding_matrix_3), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3;
gc.collect()<load_from_csv>
train_ds = MoADataset(X, y)
 test_ds = MoADataset(X_test )
submission_cv = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
<choose_model_class>
def train_ae(mode, whole_dl):
train_loss = 0
lowest_loss = np.Inf
last_epoch_with_lowest_loss = 0
if mode == 'AE_Gene':
print('Train autoencoder gene!')
 net = AE_Gene_Net().float().to(device)
 criterion = nn.MSELoss()
optimizer = torch_optimizer.RAdam(net.parameters() , lr=0.002, weight_decay=0.0000125)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer=optimizer, factor=0.5, patience=8, min_lr = 0.00005, verbose=True 
)
 else:
print('Train autoencoder cell!')
 net = AE_Cell_Net().float().to(device)
 criterion = nn.MSELoss()
optimizer = torch_optimizer.RAdam(net.parameters() , lr=0.002, weight_decay=0.00001)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer=optimizer, factor=0.25, patience=11, min_lr = 0.000025, verbose=True 
)
 net.train()
for epoch in range(N_EPOCHS):
for X_local in whole_dl:
X_local = X_local.to(device)
 optimizer.zero_grad()
output = net(X_local)
 loss = criterion(output, X_local)
 loss.backward()
optimizer.step()
train_loss += loss.item()
train_loss /= len(whole_dl)
 scheduler.step(train_loss)
 print(f'Epoch: {epoch+1:02d}/{N_EPOCHS:02d} | Train loss: {train_loss:.08f}{"(Saving model state!)" if train_loss < lowest_loss else ""}')
 if train_loss < lowest_loss:
torch.save(net.state_dict() , f'./model_{mode}.state_dict')
 lowest_loss = train_loss
last_epoch_with_lowest_loss = epoch
if last_epoch_with_lowest_loss + 50 < epoch:
print('Early stopping!')
 break
train_loss = 0
if mode == 'AE_Gene':
net = AE_Gene_Net().float().to(device)
 else:
net = AE_Cell_Net().float().to(device)
 net.load_state_dict(torch.load(f'./model_{mode}.state_dict'))
return net, lowest_loss
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<import_modules>
class AE_Gene_Net(nn.Module):
def __init__(self):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(772),
nn.utils.weight_norm(nn.Linear(772, 541)) ,
nn.ReLU6() ,
nn.BatchNorm1d(541),
nn.utils.weight_norm(nn.Linear(541, 309)) ,
nn.ReLU6() , 
)
 self.decoder = nn.Sequential(
 nn.BatchNorm1d(309),
nn.utils.weight_norm(nn.Linear(309, 541)) ,
nn.ReLU6() ,
nn.BatchNorm1d(541),
nn.utils.weight_norm(nn.Linear(541, 772)) , 
)
 def forward(self, x, mode='reconstruct'):
x = self.encoder(x)
 if mode == 'reconstruct':
x = self.decoder(x)
 return x
ae_gene_net = AE_Gene_Net().float().to(device)
 print(ae_gene_net)
 summary(ae_gene_net,(772,))
from keras.layers.normalization import BatchNormalization 
<choose_model_class>
class AE_Cell_Net(nn.Module):
def __init__(self):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(100),
nn.utils.weight_norm(nn.Linear(100, 70)) ,
nn.ReLU6() ,
nn.BatchNorm1d(70),
nn.utils.weight_norm(nn.Linear(70, 40)) ,
nn.ReLU6() , 
)
 self.decoder = nn.Sequential(
 nn.BatchNorm1d(40),
nn.utils.weight_norm(nn.Linear(40, 70)) ,
nn.ReLU6() ,
nn.BatchNorm1d(70),
nn.utils.weight_norm(nn.Linear(70, 100)) , 
)
 def forward(self, x, mode='reconstruct'):
x = self.encoder(x)
 if mode == 'reconstruct':
x = self.decoder(x)
 return x
ae_cell_net = AE_Cell_Net().float().to(device)
 print(ae_cell_net)
 summary(ae_cell_net,(100,))
input1 = Input(shape=(50,))
x = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 x = SpatialDropout1D(0.4, name='SpatialDropout' )(x)
 x = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128' )(x)
 x = Conv1D(64, kernel_size= 1, name='1D_Convolution64' )(x)
 max_pool = GlobalMaxPooling1D(name="GlobalMaxPool" )(x)
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(22, 40 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 x = concatenate([max_pool, flatten2], name='Concatenate')
 x = Dense(128, activation="relu", name='1Dense128' )(x)
 x = Dropout(0.1, name='Dropout2' )(x)
 x = BatchNormalization(name='BatchNormalization' )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model1 = Model(inputs=[input1, input2], outputs=x)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1] )<train_model>
X_whole = torch.cat(( X[:, 1:773], X_test[:, 1:773]), 0)
 whole_ds = MoADataset(X_whole)
 whole_dl = torch.utils.data.DataLoader(whole_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 ae_gene_net, ae_gene_min_loss = train_ae('AE_Gene', whole_dl )
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994)
 model1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs=4, callbacks = [clr,])
<predict_on_test>
X_whole = torch.cat(( X[:, 773:873], X_test[:, 773:873]), 0)
 whole_ds = MoADataset(X_whole)
 whole_dl = torch.utils.data.DataLoader(whole_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 ae_cell_net, ae_cell_min_loss = train_ae('AE_Cell', whole_dl )
train_pred1 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ])
 test_pred1 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ] )<choose_model_class>
X_AE_Gene = torch.zeros([X.shape[0], 309])
 X_AE_Cell = torch.zeros([X.shape[0], 40])
 X_test_AE_Gene = torch.zeros([X_test.shape[0], 309])
 X_test_AE_Cell = torch.zeros([X_test.shape[0], 40])
 with torch.no_grad() :
for i in range(0, X.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X.shape[0]:
end = X.shape[0]
else:
end = start+BATCH_SIZE
X_AE_Gene[start:end] = ae_gene_net(X[start:end, 1:773].to(device), mode='' ).cpu()
X_AE_Cell[start:end] = ae_cell_net(X[start:end, 773:873].to(device), mode='' ).cpu()
for i in range(0, X_test.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_test.shape[0]:
end = X_test.shape[0]
else:
end = start+BATCH_SIZE
X_test_AE_Gene[start:end] = ae_gene_net(X_test[start:end, 1:773].to(device), mode='' ).cpu()
X_test_AE_Cell[start:end] = ae_cell_net(X_test[start:end, 773:873].to(device), mode='' ).cpu()
X = torch.cat(( X, X_AE_Gene, X_AE_Cell), 1)
 X_test = torch.cat(( X_test, X_test_AE_Gene, X_test_AE_Cell), 1 )
input1 = Input(shape=(50,))
embedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 dropout = SpatialDropout1D(0.4, name='SpatialDropout' )(embedding)
 bidirectionLSTM = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM128' )(dropout)
 bidirectionLSTM2 = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM2' )(bidirectionLSTM)
 max_pool1 = GlobalMaxPooling1D(name="GlobalMaxPool" )(bidirectionLSTM)
 max_pool2 = GlobalMaxPooling1D(name="GlobalMaxPool2" )(bidirectionLSTM2)
 conc = Concatenate()([max_pool1, max_pool2])
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(12, 50 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 merge = concatenate([conc, flatten2], name='Concatenate')
 dense64 = Dense(128, activation="relu", name='1Dense128' )(merge)
 dropout2 = Dropout(0.1, name='Dropout2' )(dense64)
 batchnormal = BatchNormalization(name='BatchNormalization' )(dropout2)
 final = Dense(1, activation="sigmoid" )(batchnormal)
 model1 = Model(inputs=[input1, input2], outputs=final)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])
 model1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs= 4, callbacks = [clr,] )<predict_on_test>
X_length, X_test_length = X.shape[0], X_test.shape[0]
X_temp = torch.cat(( X, X_test), 0)
 print('Shape before:', X_temp.shape)
 selector = VarianceThreshold(0.5)
 X_temp = selector.fit_transform(X_temp)
 X_temp = torch.tensor(X_temp)
 print('Shape after:', X_temp.shape)
 X = X_temp[0:X_length]
X_test = X_temp[X_length:X_length+X_test_length]
train_pred2 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ])
 test_pred2 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ] )<choose_model_class>
def train_mlp(train_dl, val_dl, seed, fold):
train_loss = 0
val_loss = 0
lowest_val_loss = np.Inf
last_epoch_with_lowest_val_loss = 0
net = MLPNet().float().to(device)
 criterion = nn.BCELoss()
optimizer = torch_optimizer.AdaBelief(net.parameters() , lr=0.1, weight_decay=0.00002, rectify=False, weight_decouple=False)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(
 optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1.0, epochs=N_EPOCHS, steps_per_epoch=len(train_dl)
)
 for epoch in range(N_EPOCHS):
net.train()
for X_local, y_local in train_dl:
X_local = X_local.to(device)
 y_local = y_local.to(device)
 optimizer.zero_grad()
output = net(X_local)
 loss = criterion(output, y_local)
 loss.backward()
optimizer.step()
scheduler.step()
train_loss += loss.item()
net.eval()
for X_local, y_local in val_dl:
X_local = X_local.to(device)
 y_local = y_local.to(device)
 output = net(X_local)
 loss = criterion(output, y_local)
 val_loss += loss.item()
train_loss /= len(train_dl)
 val_loss /= len(val_dl)
 print(f'Epoch: {epoch+1:02d}/{N_EPOCHS:02d} | Train loss: {train_loss:.08f} - Val loss: {val_loss:.08f}{"(Saving model state!)" if val_loss < lowest_val_loss else ""}')
 if val_loss < lowest_val_loss:
torch.save(net.state_dict() , f'./model_mlp_{seed:02d}_{fold:02d}.state_dict')
 lowest_val_loss = val_loss
last_epoch_with_lowest_val_loss = epoch
if last_epoch_with_lowest_val_loss + 30 < epoch:
print('Early stopping!')
 break
train_loss = 0
val_loss = 0
net = MLPNet().float().to(device)
 net.load_state_dict(torch.load(f'./model_mlp_{seed:02d}_{fold:02d}.state_dict'))
return net, lowest_val_loss
input1 = Input(shape=(50,))
embedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 dropout = SpatialDropout1D(0.3, name='SpatialDropout' )(embedding)
 bidirectionLSTM = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128' )(dropout)
 x1 = Conv1D(100, activation='relu', kernel_size=1,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=110000))(bidirectionLSTM)
 x2 = Conv1D(80, activation='relu', kernel_size=2,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=120000))(bidirectionLSTM)
 x3 = Conv1D(30, activation='relu', kernel_size=3,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=130000))(bidirectionLSTM)
 x4 = Conv1D(12, activation='relu', kernel_size=5,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=140000))(bidirectionLSTM)
 x1 = GlobalMaxPooling1D()(x1)
 x2 = GlobalMaxPooling1D()(x2)
 x3 = GlobalMaxPooling1D()(x3)
 x4 = GlobalMaxPooling1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(12, 40 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 merge = concatenate([c, flatten2], name='Concatenate')
 x = Dense(128, activation="relu", name='1Dense128' )(merge)
 x = Dropout(0.1, name='Dropout2' )(x)
 x = BatchNormalization(name='BatchNormalization' )(x)
 final = Dense(1, activation="sigmoid" )(x)
 model1 = Model(inputs=[input1, input2], outputs=final)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1] )<train_model>
class MLPNet(nn.Module):
def __init__(self):
super().__init__()
self.bn1 = nn.BatchNorm1d(X.shape[1])
 self.drop1 = nn.Dropout(0.2)
 self.fc1 = nn.utils.weight_norm(nn.Linear(X.shape[1], 1280))
self.bn2 = nn.BatchNorm1d(1280)
 self.drop2 = nn.Dropout(0.5)
 self.fc2 = nn.utils.weight_norm(nn.Linear(1280, 1280))
self.bn3 = nn.BatchNorm1d(1280)
 self.drop3 = nn.Dropout(0.5)
 self.fc3 = nn.utils.weight_norm(nn.Linear(1280, y.shape[1]))
self.relu = nn.ReLU6()
self.sigmoid = nn.Sigmoid()
def forward(self, x):
x = self.bn1(x)
 x = self.drop1(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.bn2(x)
 x = self.drop2(x)
 x = self.fc2(x)
 x = self.relu(x)
 x = self.bn3(x)
 x = self.drop3(x)
 x = self.fc3(x)
 x = self.sigmoid(x)
 return x
mlp_net = MLPNet().float().to(device)
 print(mlp_net)
 summary(mlp_net,(X.shape[1],))
model1.fit([train_padded_docs, X_train_stat],
y,
batch_size=512, epochs=4, callbacks = [clr,],
)<predict_on_test>
mlp_nets = []
mlp_min_losses = []
mlp_log_losses = []
for seed in SEEDS:
set_seed(seed)
 skf = MultilabelStratifiedKFold(n_splits=N_FOLDS, random_state=seed, shuffle=True)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X, y)) :
print(f'Seed: {seed}, fold: {fold+1}/{N_FOLDS}')
 X_train, y_train = X[train_idx], y[train_idx]
train_ds = MoADataset(X_train, y_train)
 train_dl = torch.utils.data.DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 X_val, y_val = X[val_idx], y[val_idx]
val_ds = MoADataset(X_val, y_val)
 val_dl = torch.utils.data.DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
 mlp_net, mlp_min_loss = train_mlp(train_dl, val_dl, seed, fold)
 mlp_nets.append(mlp_net)
 mlp_min_losses.append(mlp_min_loss)
 y_pred = torch.zeros(y_val.shape)
 with torch.no_grad() :
for i in range(0, X_val.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_val.shape[0]:
end = X_val.shape[0]
else:
end = start+BATCH_SIZE
y_pred[start:end] = mlp_net(X_val[start:end].to(device)).cpu()
mlp_log_losses.append(log_loss(y_val, y_pred))
print('='*82)
 print('Training Ended! ' )
train_pred4 = model1.predict([train_padded_docs, X_train_stat], batch_size= 512, verbose=1, callbacks = [clr, ])
 test_pred4 = model1.predict([test_padded_docs, test_data1], batch_size= 512, verbose=1, callbacks=[clr, ] )<compute_test_metric>
print(f'AE Gene loss: {ae_gene_min_loss}')
 print(f'AE Cell loss: {ae_cell_min_loss}')
 print('MLP CV min loss:')
 [print(l)for l in mlp_min_losses]
for i in range(len(SEEDS)) :
print(f'MLP average CV min loss(SEED {SEEDS[i]}): {sum(mlp_min_losses[i*N_FOLDS:i*N_FOLDS+N_FOLDS])/N_FOLDS}')
 print(f'MLP average CV min loss: {sum(mlp_min_losses)/len(mlp_min_losses)}')
 print('MLP log loss:')
 [print(l)for l in mlp_log_losses]
for i in range(len(SEEDS)) :
print(f'MLP average CV log loss(SEED {SEEDS[i]}): {sum(mlp_log_losses[i*N_FOLDS:i*N_FOLDS+N_FOLDS])/N_FOLDS}')
 print(f'MLP average CV log loss: {sum(mlp_log_losses)/len(mlp_log_losses)}' )
pred_val_y = 0.3 * train_pred1 + 0.3 * train_pred2 + 0.4 * train_pred4
pred_test_y = 0.3 * test_pred1 + 0.3 * test_pred2 + 0.4 * test_pred4
thresholds = []
for thresh in np.arange(0.1, 0.5, 0.01):
thresh = np.round(thresh, 2)
 res = f1_score(y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<data_type_conversions>
with torch.no_grad() :
for i in range(0, X_test.shape[0], BATCH_SIZE):
start = i
if start+BATCH_SIZE >= X_test.shape[0]:
end = X_test.shape[0]
else:
end = start+BATCH_SIZE
test_results = torch.zeros([end-start, y.shape[1]])
 for mlp_net in mlp_nets:
test_results += mlp_net(X_test[start:end].to(device)).cpu()
test_results /= int(N_FOLDS * len(SEEDS))
df_submission.iloc[start:end, 1:] = test_results
pred_test_y =(pred_test_y > best_thresh ).astype(int )<prepare_output>
df_submission.to_csv('submission.csv', index=False )