Kaggle competition: Quora Insincere Questions Classification
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 test.head()<count_missing_values>
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model
train.isnull().sum()<count_missing_values>
NEG_INF = -10000
TINY_FLOAT = 1e-6
def mask_mean(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 mask_sum = torch.sum(
 seq * mask.unsqueeze(-1 ).float() , dim=1)
 seq_len = torch.sum(mask, dim=-1)
 mask_mean = mask_sum /(seq_len.unsqueeze(-1 ).float() + TINY_FLOAT)
 return mask_mean
def mask_max(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 torch
mask_max, _ = torch.max(
 seq +(1 - mask.unsqueeze(-1 ).float())* NEG_INF,
dim=1)
 return mask_max
def seq_mask(seq_len, max_len):

 idx = torch.arange(max_len ).to(seq_len ).repeat(seq_len.size(0), 1)
 mask = torch.gt(seq_len.unsqueeze(1), idx ).to(seq_len)
 return mask
class DynamicLSTM(nn.Module):

 def __init__(self, input_size, hidden_size=100,
num_layers=1, dropout=0., bidirectional=False):
super(DynamicLSTM, self ).__init__()
self.lstm = nn.LSTM(
 input_size, hidden_size, num_layers, bias=True,
batch_first=True, dropout=dropout, bidirectional=bidirectional)
 def forward(self, x, seq_lens):
_, idx_sort = torch.sort(seq_lens, dim=0, descending=True)
 _, idx_unsort = torch.sort(idx_sort, dim=0)
 x_sort = torch.index_select(x, dim=0, index=idx_sort)
 seq_lens_sort = torch.index_select(seq_lens, dim=0, index=idx_sort)
 x_packed = pack_padded_sequence(
 x_sort, seq_lens_sort, batch_first=True)
 y_packed, _ = self.lstm(x_packed)
 y_sort, length = pad_packed_sequence(y_packed, batch_first=True)
 y = torch.index_select(y_sort, dim=0, index=idx_unsort)
 return y
class QuoraModel(nn.Module):

 def __init__(self, args):
super(QuoraModel, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
embed_dim = 300
num_classes = 1
num_layers = 2
hidden_dim = 50
dropout = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.rnn = DynamicLSTM(
 embed_dim, hidden_dim, num_layers=num_layers,
dropout=dropout, bidirectional=True)
 self.fc = nn.Linear(hidden_dim * 4, hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(dropout)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq, seq_len):
max_seq_len = torch.max(seq_len)
 mask = seq_mask(seq_len, max_seq_len)
 e = self.drop(self.embed(word_seq))
r = self.rnn(e, seq_len)
 r_avg = mask_mean(r, mask)
 r_max = mask_max(r, mask)
 r = torch.cat([r_avg, r_max], dim=-1)
 f = self.drop(self.act(self.fc(r)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator: 
(text, text_len), target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text, text_len)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(*text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(*text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred
train.isnull().sum()<count_missing_values>
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'
test.isnull().sum()<count_missing_values>
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length,
include_lengths=True)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict
test.isnull().sum()<create_dataframe>
args = pre()
pd.pivot_table(train, index = 'Survived', values = ['Age','SibSp','Parch','Fare'] )<create_dataframe>
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = QuoraModel(model_args)
 trainer_args = {
"epochs": 3,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )
pd.pivot_table(train, index = 'Survived', columns = 'Pclass', values = 'Ticket' ,aggfunc ='count' )<create_dataframe>
train(**args )
pd.pivot_table(train, index = 'Survived', columns = 'Sex', values = 'Ticket' ,aggfunc ='count' )<create_dataframe>
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {} threshold: {:>.2f}".format(best_f1, best_thresh))
print('')
 return args
pd.pivot_table(train, index = 'Survived', columns = 'Embarked', values = 'Ticket' ,aggfunc ='count' )<feature_engineering>
def infer(**args):

 print("Predicting...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 predictor = Predictor(batch_size=128, use_cuda=False)
 data_test = args["data_test"]
threshold = args["threshold"]
y_pred = predictor.predict(model, data_test, threshold=threshold)
 test_df = pd.read_csv(test_path, index_col=False, header=0)
 data = {"qid": test_df["qid"], "prediction": y_pred}
subm_df = pd.DataFrame(data=data)
 subm_df.to_csv(submission_path, header=True, index=False)
 print("submission saved as {}.".format(submission_path))
print('' )