Kaggle competition: Quora Insincere Questions Classification
train_images = np.concatenate(( X_train_keras,X_train,X_test_keras), axis=0)
 print("new Concatenated train_images ", train_images.shape)
 print("_"*50)
 train_labels = np.concatenate(( train_labels_keras,y_train,test_labels_keras), axis=0)
 print("new Concatenated train_labels ", train_labels.shape )<normalization>
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model
scale = np.max(train_images)
 train_images /= scale
X_test /= scale
print("Max: {}".format(scale)) 
<train_model>
NEG_INF = -10000
TINY_FLOAT = 1e-6
def mask_softmax(matrix, mask=None):

 if mask is None:
result = F.softmax(matrix, dim=-1)
 else:
mask_norm =(( 1 - mask)* NEG_INF ).to(matrix)
 for i in range(matrix.dim() - mask_norm.dim()):
mask_norm = mask_norm.unsqueeze(1)
 result = F.softmax(matrix + mask_norm, dim=-1)
 return result
def mask_mean(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 mask_sum = torch.sum(
 seq * mask.unsqueeze(-1 ).float() , dim=1)
 seq_len = torch.sum(mask, dim=-1)
 mask_mean = mask_sum /(seq_len.unsqueeze(-1 ).float() + TINY_FLOAT)
 return mask_mean
def mask_max(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 torch
mask_max, _ = torch.max(
 seq +(1 - mask.unsqueeze(-1 ).float())* NEG_INF,
dim=1)
 return mask_max
def seq_mask(seq_len, max_len):

 idx = torch.arange(max_len ).to(seq_len ).repeat(seq_len.size(0), 1)
 mask = torch.gt(seq_len.unsqueeze(1), idx ).to(seq_len)
 return mask
class DynamicLSTM(nn.Module):

 def __init__(self, input_size, hidden_size=100,
num_layers=1, dropout=0., bidirectional=False):
super(DynamicLSTM, self ).__init__()
self.lstm = nn.LSTM(
 input_size, hidden_size, num_layers, bias=True,
batch_first=True, dropout=dropout, bidirectional=bidirectional)
 def forward(self, x, seq_lens):
_, idx_sort = torch.sort(seq_lens, dim=0, descending=True)
 _, idx_unsort = torch.sort(idx_sort, dim=0)
 x_sort = torch.index_select(x, dim=0, index=idx_sort)
 seq_lens_sort = torch.index_select(seq_lens, dim=0, index=idx_sort)
 x_packed = pack_padded_sequence(
 x_sort, seq_lens_sort, batch_first=True)
 y_packed, _ = self.lstm(x_packed)
 y_sort, length = pad_packed_sequence(y_packed, batch_first=True)
 y = torch.index_select(y_sort, dim=0, index=idx_unsort)
 return y
class QuoraModel(nn.Module):

 def __init__(self, args):
super(QuoraModel, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
embed_dim = 300
num_classes = 1
num_layers = 2
hidden_dim = 50
dropout = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.rnn = DynamicLSTM(
 embed_dim, hidden_dim, num_layers=num_layers,
dropout=dropout, bidirectional=True)
 self.fc_att = nn.Linear(hidden_dim * 2, 1)
 self.fc = nn.Linear(hidden_dim * 6, hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(dropout)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq, seq_len):
max_seq_len = torch.max(seq_len)
 mask = seq_mask(seq_len, max_seq_len)
 e = self.drop(self.embed(word_seq))
r = self.rnn(e, seq_len)
 att = self.fc_att(r ).squeeze(-1)
 att = mask_softmax(att, mask)
 r_att = torch.sum(att.unsqueeze(-1)* r, dim=1)
 r_avg = mask_mean(r, mask)
 r_max = mask_max(r, mask)
 r = torch.cat([r_avg, r_max, r_att], dim=-1)
 f = self.drop(self.act(self.fc(r)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator: 
(text, text_len), target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text, text_len)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(*text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(*text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred
input_size = train_images.shape
n_logits = train_labels.shape[1]
print("Input: {}".format(input_size))
print("Output: {}".format(n_logits))<init_hyperparams>
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 30
batch_size = 512
X_train, X_val, Y_train, Yval = train_test_split(train_images, train_labels, train_size = 0.90)
 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length,
include_lengths=True)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict
model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3),padding='same', activation='relu', input_shape=input_size[1:]))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None))
model.add(tf.keras.layers.Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(n_logits, activation='softmax'))
optimizer = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 model.summary()<train_model>
args = pre()
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Yval),
verbose = 1, steps_per_epoch=X_train.shape[0] )<save_to_csv>
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = QuoraModel(model_args)
 trainer_args = {
"epochs": 4,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )
predictions = model.predict_classes(X_test, verbose=0)
 print(predictions)
 pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) , "Label": predictions} ).to_csv("preds.csv", index=False, header=True)
<install_modules>
train(**args )
!pip install keras-tuner<data_type_conversions>
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {} threshold: {:>.2f}".format(best_f1, best_thresh))
print('')
 return args
<load_pretrained><EOS> 
def infer(**args):

 print("Predicting...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 predictor = Predictor(batch_size=128, use_cuda=False)
 data_test = args["data_test"]
threshold = args["threshold"]
y_pred = predictor.predict(model, data_test, threshold=threshold)
 test_df = pd.read_csv(test_path, index_col=False, header=0)
 data = {"qid": test_df["qid"], "prediction": y_pred}
subm_df = pd.DataFrame(data=data)
 subm_df.to_csv(submission_path, header=True, index=False)
 print("submission saved as {}.".format(submission_path))
print('' )