Kaggle competition: Quora Insincere Questions Classification
def train_cross_validate(folds = 5):
histories = []
models = []
kfold = KFold(folds, shuffle = True, random_state = SEED)
 for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('
print('
print('
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 if MODEL_SELECT is 'Ensamble' or MODEL_SELECT is 'Model1':
model1 = get_model1()
history1 = model1.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback, es_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model1)
 histories.append(history1)
 if MODEL_SELECT is 'Ensamble' or MODEL_SELECT is 'Model2':
model2 = get_model2()
history2 = model2.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback, es_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model2)
 histories.append(history2)
 return histories, models<find_best_model_class>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )
def train_and_predict(folds = 5):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training %i folds'%folds)
 histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(len(models)) ], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 return histories, models<predict_on_test>
print('Preproccesing texts.... ')
 print('lower...')
 df["question_text"] = df["question_text"].apply(lambda x: x.lower())
 df_final["question_text"] = df_final["question_text"].apply(lambda x: x.lower())
 contraction_mapping = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have" }
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
print('contractions...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
punct_mapping = {"‘": "'",
"₹": "e",
"´": "'",
"°": "",
"€": "e",
"™": "tm",
"√": " sqrt ",
"×": "x",
"²": "2",
"—": "-",
"–": "-",
"’": "'",
"_": "-",
"`": "'",
'“': '"',
'”': '"',
'“': '"',
"£": "e",
'∞': 'infinity',
'θ': 'theta',
'÷': '/',
'α': 'alpha',
'•': '.',
'à': 'a',
'−': '-',
'β': 'beta',
'∅': '',
'³': '3',
'π': 'pi',
}
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
print('clean special chars...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
mispell_dict = {'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization'}
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
print('clean misspellings...')
 df["question_text"] = df["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))
df_final["question_text"] = df_final["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))
histories, models = train_and_predict(folds = FOLDS )<split>
dim = 300
num_words = 75966
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(list(df['question_text'])+list(df_final['question_text']))
print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape)
 x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )
all_labels = []; all_prob = []; all_pred = []
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for j,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('Inferring fold',j+1,'validation images...')
 VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES'])
 NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)
 cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))
images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
all_labels.append(next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy())
 prob = models[j].predict(images_ds)
 all_prob.append(prob)
 all_pred.append(np.argmax(prob, axis=-1))
cm_correct_labels = np.concatenate(all_labels)
 cm_probabilities = np.concatenate(all_prob)
 cm_predictions = np.concatenate(all_pred)
 print("Correct labels: ", cm_correct_labels.shape, cm_correct_labels)
 print("Predicted labels: ", cm_predictions.shape, cm_predictions )<compute_test_metric>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<install_modules>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_para.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
!pip install -q efficientnet<import_modules>
matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()
print("Tensorflow version " + tf.__version__ )<set_options>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
IMAGE_SIZE = [512, 512]
EPOCHS = 20
BATCH_SIZE = 32 * strategy.num_replicas_in_sync<choose_model_class>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
best threshold is % f with score %f'%(best_threshold,best_score))
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
class Sentiment(nn.Module):
def __init__(self,matrix,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',matrix.shape)
 self.batch_size = batch_size
self.hidden_dim = 64
self.lin_dim = 32
self.n_layers = 1
self.embedding = nn.Embedding(matrix.shape[0],matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=matrix.shape[1],
hidden_size=self.hidden_dim,
num_layers=self.n_layers,
bidirectional=True,
batch_first=True)
 self.gru = nn.GRU(input_size=matrix.shape[1],
hidden_size=self.hidden_dim,
num_layers=self.n_layers,
bidirectional=True,
batch_first=True)
 self.linear1 = nn.Linear(4*self.hidden_dim,self.lin_dim)
 self.relu = nn.ReLU()
self.linear2 = nn.Linear(self.lin_dim,1)
 self.dropout = nn.Dropout(0.1)
 def forward(self,x):
hidden =(torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 hidden_gru = torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda()
e = self.embedding(x)
 lstm_out, hidden = self.lstm(e, hidden)
 gru_out, hidden_gru = self.gru(e, hidden_gru)
 out = torch.cat([hidden[0][-2,:,:], hidden[0][-1,:,:],hidden_gru[-2,:,:], hidden_gru[-1,:,:]], dim=1 ).cuda()
out = self.linear1(out)
 out = self.relu(out)
 out = self.dropout(out)
 return self.linear2(out)
 def train(model, train_loader):
loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 for epoch in range(3):
model.train()
avg_loss = 0
for batch,(x_batch,y_true)in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(x_batch ).squeeze(1)
 loss = loss_function(y_pred,y_true)
 avg_loss += loss.item()
loss.backward()
optimizer.step()
print('EPOCH: ',epoch,': ',avg_loss/batch)
 print('-'*80)
 print('Training finished.... ')
 def eval_on_set(test_loader):
pred = []
with torch.no_grad() :
model.eval()
for x_test_batch,y_test_batch in list(test_loader):
pred += torch.sigmoid(model(x_test_batch ).squeeze(1)).cpu().detach().numpy().tolist()
return np.array(pred)
 def eval_sub(submission_loader):
pred = []
with torch.no_grad() :
model.eval()
for(x,)in list(submission_loader):
y_pred = torch.sigmoid(model(x ).squeeze(1)).detach()
pred += y_pred.cpu().numpy().tolist()
return np.array(pred)
 batch_size = 512
print('Batch size = ',batch_size)
 submission_dataset = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long ).cuda())
 submission_loader = torch.utils.data.DataLoader(dataset=submission_dataset,batch_size=batch_size, shuffle=False)
 patience = 2
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(x_test.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=6017 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
x_train_tensor = torch.tensor(X_train1, dtype=torch.long ).cuda()
y_train_tensor = torch.tensor(y_train1, dtype=torch.float32 ).cuda()
train_dataset = torch.utils.data.TensorDataset(x_train_tensor,y_train_tensor)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
 x_val_tensor = torch.tensor(X_val, dtype=torch.long ).cuda()
y_val_tensor = torch.tensor(y_val, dtype=torch.float32 ).cuda()
val_dataset = torch.utils.data.TensorDataset(x_val_tensor,y_val_tensor)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=False)
 model = Sentiment(matrix, batch_size=batch_size ).cuda()
train(model,train_loader)
 y_pred = eval_on_set(val_loader)
 train_meta[valid_idx] = y_pred.reshape(-1)
 y_pred = eval_sub(submission_loader)
 test_meta += y_pred.reshape(-1)/ len(splits)
 print('FINISHED TRAINING META...' )
<define_variables><EOS> 
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] =(test_meta > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )