Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<compute_test_metric>
FOLD = 0
SEED = 1337
NOTIFY_EACH_EPOCH = False
WORKERS = 0
BATCH_SIZE = 512
N_SPLITS = 10
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def save_checkpoint(state):

 print(" Saving checkpoint")
 filename = f'./checkpoint-{state["epoch"]}.pt.tar'
torch.save(state, filename)
 def initialize(model, path=None, optimizer=None):
if path == None:
checkpoints = glob.glob('./*.pt.tar')
 path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0])for checkpoint in checkpoints])]
checkpoint = torch.load(path)
 model.load_state_dict(checkpoint['model'])
 print(f' Loaded checkpoint {path} | Trained for {checkpoint["epoch"] + 1} epochs')
 if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
 epoch = checkpoint['epoch'] + 1
train_iteration = checkpoint['train_iteration']
val_iteration = checkpoint['val_iteration']
return model, optimizer, epoch, train_iteration, val_iteration
else:
return model
feature_importance(rnd_search_rf.best_estimator_, feature_names, 10 )<find_best_params>
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
final_rf_clf = rnd_search_rf.best_estimator_
final_rf_clf<predict_on_test>
kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_idx, val_idx = list(kfold.split(train)) [FOLD]
x_train, x_val = train.iloc[train_idx], train.iloc[val_idx]
x_train.to_csv('train.csv')
 x_val.to_csv('val.csv' )
y_train_pred_rf = final_rf_clf.predict(X_train )<compute_test_metric>
nlp = English()
def tokenize(sentence):
x = nlp(sentence)
 return [token.text for token in x]
metric_scores(y_train, y_train_pred_rf )<predict_on_test>
question_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)
 target_field = data.Field(sequential=False, use_vocab=False, batch_first=True)
 train_fields = [ 
('id', None), 
('qid', None), 
('question_text', question_field), 
('target', target_field)
 ]
test_fields = [ 
('qid', None), 
('question_text', question_field)
 ]
train_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)
 test_dataset = data.TabularDataset('.. /input/test.csv', format='CSV', skip_header=True, fields=test_fields)
 vectors = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 question_field.build_vocab(train_dataset, max_size=95000)
 question_field.vocab.set_vectors(vectors.stoi, vectors.vectors, vectors.dim)
 pretrained_embedding = question_field.vocab.vectors
train_dataloader, val_dataloader = data.BucketIterator.splits(( train_dataset, val_dataset),(BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)
 test_dataloader = data.BucketIterator(test_dataset, 1, sort=False, shuffle=False)
 print(f'Train Dataset: {len(train_dataset)}')
 print(f'Val Dataset: {len(val_dataset)}')
 print(f'Test Dataset: {len(test_dataset)}' )
y_pred_rf = final_rf_clf.predict(X_test )<prepare_output>
class SelfAttention(nn.Module):
def __init__(self, hidden_size, batch_first=False):
super(SelfAttention, self ).__init__()
self.hidden_size = hidden_size
self.batch_first = batch_first
self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)
 stdv = 1.0 / math.sqrt(self.hidden_size)
 for weight in self.att_weights:
nn.init.uniform_(weight, -stdv, stdv)
 def get_mask(self):
pass
def forward(self, inputs, lengths):
if self.batch_first:
batch_size, max_len = inputs.size() [:2]
else:
max_len, batch_size = inputs.size() [:2]
weights = torch.bmm(inputs,
self.att_weights 
.permute(1, 0)
.unsqueeze(0)
.repeat(batch_size, 1, 1)
)
 attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)
 mask = torch.ones(attentions.size() , requires_grad=True ).cuda()
for i, l in enumerate(lengths):
if l < max_len:
mask[i, l:] = 0
masked = attentions * mask
_sums = masked.sum(-1 ).unsqueeze(-1)
 attentions = masked.div(_sums)
 weighted = torch.mul(inputs, attentions.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations, attentions
class BaselineLSTM(nn.Module):
def __init__(self, embedding):
super(BaselineLSTM, self ).__init__()
self.embedding = nn.Embedding.from_pretrained(embedding)
 self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=2, batch_first=True, bidirectional=True)
 self.attention = SelfAttention(128*2, batch_first=True)
 self.fc = nn.Linear(128*2, 1)
 self.logit = nn.Linear(1, 1)
 def forward(self,x, x_len):
x = self.embedding(x)
 x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
 out,(hidden, _)= self.lstm(x)
 x, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
 x, _ = self.attention(x, lengths)
 x = self.fc(x)
 x = self.logit(x ).view(-1)
 return x
survived_rf=pred_to_df(y_pred_rf)
 survived_rf.head()<save_to_csv>
start_epoch = 0
epochs = 7
early_stopping = 10
train_iteration = 0
val_iteration = 0
threshold = 0.35
model = BaselineLSTM(pretrained_embedding ).to(device)
 optimizer = optim.Adam(model.parameters())
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)
 criterion = nn.BCEWithLogitsLoss()
get_n_params(model )
save_to_csv(survived_rf,'survived_rf' )<train_on_grid>
print(f' Training Finished | Best Epoch {best_epoch} | LR: {optimizer.param_groups[0]["lr"]}
Best Train Loss: {best_train_loss} | Best Val Loss: {best_val_loss} | Best Train F1: {round(best_train_f1, 4)} | Best Val F1: {round(best_val_f1, 4)}' )
param_distribs = {
'max_depth': randint(low=1, high=5),
'n_estimators': randint(low=5, high=120),
'max_features': randint(low=5, high=15),
}
gb_clf = GradientBoostingClassifier()
rnd_search_gb = RandomizedSearchCV(gb_clf, param_distributions=param_distribs,
n_iter=40, cv=5, scoring="accuracy", random_state=42, return_train_score=True)
 rnd_search_gb.fit(X_train, y_train )<compute_test_metric>
model = BaselineLSTM(pretrained_embedding ).to(device)
 model = initialize(model)
 preds = []
model.eval()
with torch.no_grad() :
for i, batch in tqdm(enumerate(test_dataloader), total=int(len(test_dataset)/ BATCH_SIZE)) : 
(question, length)= batch.question_text
question = question.to(device)
 out = model(question, length)
 out = torch.sigmoid(out)
 pred = out.detach().cpu().numpy()
preds.append(pred)
 preds = np.concatenate(preds, axis=0 ).reshape(-1, 1)
 preds =(preds > best_threshold ).astype(int )
print('Mean training score: ' + str(( rnd_search_gb.cv_results_['mean_train_score'][rnd_search_gb.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_gb.cv_results_['std_train_score'][0]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( rnd_search_gb.cv_results_['mean_test_score'][rnd_search_gb.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_gb.cv_results_['std_test_score'][0]*100 ).round(2)) + '%)' )<find_best_params>
sample_submission['prediction'] = preds
mlc.kaggle.save_sub(sample_submission, 'submission.csv' )