Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<prepare_x_and_y>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras import regularizers
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
chunkStart = time.time()
x_train = train_df[TEXT_COLUMN].astype(str ).apply(remove_id_words)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str ).apply(remove_id_words)
 print('CHUNK', time.time() - chunkStart)
 print('OVERALL', time.time() - absStart)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False)
 MAX_FEATURES = 400000
tokenizer = text.Tokenizer(num_words = MAX_FEATURES, filters='',lower=False)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
MAX_FEATURES = MAX_FEATURES or len(tokenizer.word_index)+ 1
MAX_FEATURES
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()
checkpoint_predictions = []
weights = []
print('CHUNK', time.time() - chunkStart)
 print('OVERALL', time.time() - absStart )<train_model>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
chunkStart = time.time()
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 print('EPOCH', time.time() - chunkStart)
 print('MODEL', time.time() - chunkStart)
<save_to_csv>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False)
 print('TOTAL TIME')
 print(time.time() - absStart )<import_modules>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3 , all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix1[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )
import pandas as pd
import numpy as np
import nltk
import re
from keras.preprocessing import text, sequence
from sklearn.model_selection import train_test_split
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
import time
import random
import gc
import os
import multiprocessing as mp
import matplotlib.pyplot as plt<load_from_csv>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE,
encoding="utf8", errors='ignore'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3, all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix2[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )
def load_data(size=4):
start = time.time()
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train = clean_text(train)
 test = clean_text(test)
 x_train = train['comment_text']
y_train = train['target']
x_test = test['comment_text']
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
print('Data was successfully read in and preprocessed.Time taken: ',
round(( time.time() - start)/60, 2), ' minutes')
 return x_train, x_test, y_train, y_aux_train, test<string_transform>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3, all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix3[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )
def tokenize_text(x_train, x_test, pad_len):
MAX_WORDS = 100_000
start = time.time()
tokenizer = text.Tokenizer(num_words = MAX_WORDS)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_train = sequence.pad_sequences(x_train, maxlen=pad_len)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_test = sequence.pad_sequences(x_test, maxlen=pad_len)
 print('Text has been tokenized in', round(( time.time() - start)/60, 2), ' minutes')
 return x_train, x_test, tokenizer 
<categorify>
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2, embedding_matrix3],axis=1)
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path, encoding="utf8")as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def make_embedder(tokenizer):
GLOVE_PATH = '.. /input/crawl300d2m/crawl-300d-2M.vec'
CRAWL_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
start = time.time()
glove_matrix, unknown_words = build_matrix(tokenizer.word_index, GLOVE_PATH)
 crawl_matrix, unknown_words = build_matrix(tokenizer.word_index, CRAWL_PATH)
 embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis = -1)
 gc.collect()
print('Embedding matrix was successfully made.Time taken:',
round(( time.time() - start)/60,2), ' minutes')
 return embedding_matrix<train_model>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=True )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
def make_train_test(x_train, x_test, y_train, y_aux_train):
x_train_torch = torch.tensor(x_train, dtype=torch.long ).cuda()
x_test_torch = torch.tensor(x_test, dtype=torch.long ).cuda()
y_train_torch = torch.tensor(
 np.hstack([y_train[:, np.newaxis], y_aux_train]),
dtype=torch.float32 ).cuda()
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 test_dataset = data.TensorDataset(x_test_torch)
 print('Pytorch data has been made.')
 return train_dataset, test_dataset, y_train_torch<normalization>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets, LSTM_UNITS, max_features):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<choose_model_class>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 f_scores = []
for thresh in np.arange(0.1, 0.9, 0.01):
thresh = np.round(thresh, 2)
 f_score = metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))
f_scores.append(( f_score, thresh))
fmax, opt_thresh = max(f_scores)
 print("F1 score at threshold {0} is {1}".format(opt_thresh, fmax))
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train, test, loss_fn, output_dim,
lr=0.001, batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<define_variables>
pred_all_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 seed_everything()<categorify>
pred_test_y = pred_all_test_y
pred_test_y =(pred_test_y>opt_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )