Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
tqdm.pandas()
%matplotlib inline
sns.set_style('whitegrid')
class My_Dataset(torch.utils.data.Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
return
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx])for key, val in self.encodings.items() }
if self.labels is not None and len(self.labels)>0:
item['labels'] = self.labels[idx]
return item
def __len__(self):
return len(self.labels )<create_dataframe>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
def get_mapType_Dataset(ds_names, isLabeledDataset):
input_ids = []
attention_mask = []
labels = []
lang = []
dataset_iter = MyIterableDataset(ds_names, isLabeledDataset)
 for encoded_dict in dataset_iter:
input_ids.append(encoded_dict['input_ids'])
 attention_mask.append(encoded_dict['attention_mask'])
 labels.append(encoded_dict['labels'])
 encodings = {'input_ids':input_ids, 'attention_mask':attention_mask}
my_dataset = My_Dataset(encodings, labels)
 return my_dataset<create_dataframe>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2 )
eval_dataset = get_mapType_Dataset(eval_ds_names, True)
 test_dataset = get_mapType_Dataset(test_ds_names, False )<load_pretrained>
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
def save_my_model_and_tokenizer(model, tokenizer=None, output_dir='./model_save/'):

 model_state_file = output_dir + 'my_model.bin'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 print("Saving model to %s" % model_state_file)
 model.cpu()
state = model.state_dict()
torch.save(state, model_state_file)
 if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
 model.cuda()
return<load_pretrained>
embed_size = 300
max_features = 100000
maxlen = 70
def load_model(bert_model_name, output_dir='./model_save/'):
model_state_file = output_dir + 'my_model.bin'
model = MyGAPModelForSeqClf(bert_model_name)
 state = torch.load(model_state_file)
 model.load_state_dict(state)
 return model<categorify>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<compute_test_metric>
train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
def compute_metrics(labels, pred_logits):
preds = pred_logits.argmax(-1)
 precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
 acc = accuracy_score(labels, preds)
 mcc = matthews_corrcoef(labels, preds)
 metrics = {
'mcc': mcc,
'accuracy': acc,
'f1': f1,
'precision': precision,
'recall': recall
}
return metrics<train_model>
train_y = train_df['target'].values
val_y = val_df['target'].values
class MyTrainer:
def __init__(self, model, args, train_dataset, eval_dataset, tokenizer, compute_metrics=compute_metrics):
self.model = model
self.args = args
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.tokenizer = tokenizer
self.compute_metrics = compute_metrics
self.isTrained = False
self.device = self.get_device_type()
self.optimizer = AdamW(model.parameters() ,
lr = args.learning_rate,
eps = args.adam_epsilon 
)
 self.epochs = self.args.num_train_epochs
self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders()
return
def get_device_type(self):
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 return device
def get_dataLoaders(self):
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = SequentialSampler(self.train_dataset)
 train_dataloader = DataLoader(
 self.train_dataset,
sampler = train_sampler,
batch_size = self.args.per_device_train_batch_size 
)
 validation_dataloader = DataLoader(
 self.eval_dataset,
sampler = SequentialSampler(self.eval_dataset),
batch_size = self.args.per_device_eval_batch_size 
)
 num_training_steps = len(train_dataloader)* self.epochs
lr_scheduler = get_linear_schedule_with_warmup(self.optimizer,
num_warmup_steps = self.args.warmup_steps,
num_training_steps = num_training_steps)
 return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps
def test_iterate_dataloader() :
for step, batch in enumerate(self.train_dataloader):
print(step, batch)
 return
def train(self, is_train_base_encoding_model=True):
if is_train_base_encoding_model:
self.model.unfreez()
else:
self.model.freez()
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
min_val_loss = 99999999
for epoch_i in range(0, self.epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, self.epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
self.model.train()
for step, batch in enumerate(self.train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(self.train_dataloader), elapsed))
if(self.args.max_steps > 0 and self.args.max_steps < step)or \ 
(is_train_base_encoding_model and self.args.eval_steps>0 and step % self.args.eval_steps==0 and step>0):
avg_train_loss = total_train_loss / step
training_time = format_time(time.time() - t0)
 print("Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
save_my_model_and_tokenizer(self.model, self.tokenizer, output_dir='./model_save/')
 if self.args.max_steps > 0 and self.args.max_steps < step:
return training_stats
self.model.zero_grad()
for k in batch:
batch[k] = batch[k].to(self.device)
 output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters() , 1.0)
 self.optimizer.step()
self.lr_scheduler.step()
avg_train_loss = total_train_loss / len(self.train_dataloader)
 training_time = format_time(time.time() - t0)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("
Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
save_my_model_and_tokenizer(self.model, self.tokenizer, output_dir='./model_save/')
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.plot_train_stats(training_stats)
 return training_stats
def evaluate(self):
t0 = time.time()
self.model.eval()
total_eval_mcc = 0
total_eval_f1 = 0
total_eval_precision = 0
total_eval_recall = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in self.validation_dataloader:
with torch.no_grad() :
for k in batch:
batch[k] = batch[k].to(self.device)
 output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = batch['labels'].to('cpu' ).numpy()
metrics = self.compute_metrics(label_ids, logits)
 total_eval_mcc += metrics['mcc']
total_eval_f1 += metrics['f1']
total_eval_precision += metrics['precision']
total_eval_recall += metrics['recall']
total_eval_accuracy += metrics['accuracy']
avg_val_f1 = total_eval_f1 / len(self.validation_dataloader)
 print(" F1: {0:.3f}".format(avg_val_f1))
avg_val_mcc = total_eval_mcc / len(self.validation_dataloader)
 print(" MCC: {0:.3f}".format(avg_val_mcc))
avg_val_precision = total_eval_precision / len(self.validation_dataloader)
 print(" Precision: {0:.3f}".format(avg_val_precision))
avg_val_recall = total_eval_recall / len(self.validation_dataloader)
 print(" Recall: {0:.3f}".format(avg_val_recall))
avg_val_accuracy = total_eval_accuracy / len(self.validation_dataloader)
 print(" Accuracy: {0:.3f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(self.validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time
def plot_train_stats(self, training_stats):
mccs = [e['eval_mcc'] for e in training_stats]
accuracies = [e['eval_accuracy'] for e in training_stats]
f1_scores = [e['eval_f1'] for e in training_stats]
precisions = [e['eval_precision'] for e in training_stats]
recalls = [e['eval_recall'] for e in training_stats]
losses = [e['eval_loss'] for e in training_stats]
epochs = training_stats[-1]['epoch']
print('mccs:', mccs)
 print('accuracies:', accuracies)
 print('precisions:', precisions)
 print('recalls:', recalls)
 print('f1_scores:', f1_scores)
 print('losses:', losses)
 sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_mcc')
 sns.lineplot(x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')
 sns.lineplot(x=np.arange(1, epochs + 1), y=precisions, label='val_precision')
 sns.lineplot(x=np.arange(1, epochs + 1), y=recalls, label='val_recall')
 sns.lineplot(x=np.arange(1, epochs + 1), y=f1_scores, label='val_f1')
 plt.show()
return
def getTrainedModel(self):
if self.isTrained:
return self.model
return None
def predict(self, prediction_dataset, isRemoveLabels=True):

 prediction_sampler = SequentialSampler(prediction_dataset)
 prediction_dataloader = DataLoader(prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)
 print('Predicting labels for {:,} test sentences...'.format(len(prediction_dataset)))
 self.model.eval()
predictions = []
for batch in prediction_dataloader:
batch = {t:batch[t].to(self.device)for t in batch}
with torch.no_grad() :
if isRemoveLabels:
batch.pop('labels')
 for k in batch:
batch[k] = batch[k].to(self.device)
 outputs = model(**batch)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')
 print('Done prediction')
 pred_logits = np.concatenate(predictions, axis=0)
 return pred_logits, None, None<choose_model_class>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
class MyGAPModelForSeqClf(nn.Module):
def __init__(self, bert_model_name, outputCount=3, drop_prob=0.1, nonlin=nn.ReLU()):
super(MyGAPModelForSeqClf, self ).__init__()
self.model = AutoModelForSequenceClassification.from_pretrained(bert_model_name ).base_model
self.drop_prob = drop_prob
self.nonlin = nonlin
self.outputCount = outputCount
hidden_size = self.model.config.hidden_size
self.dense = nn.Linear(hidden_size, hidden_size)
 self.batchnorm = nn.BatchNorm1d(hidden_size)
 self.outDense = nn.Linear(hidden_size, outputCount)
 self.dropout = nn.Dropout(drop_prob)
 self.outActivtn = nn.LogSoftmax(dim=1)
 self.NLLLoss = nn.NLLLoss()
return
def freez(self):
for param in self.model.base_model.parameters() :
param.requires_grad = False
return
def unfreez(self):
for param in self.model.base_model.parameters() :
param.requires_grad = True
return
def forward(self, input_ids, attention_mask, token_type_ids=None, labels=None, **kwargs):
last_hidden_states = None
if token_type_ids is None:
moutput = self.model(input_ids=input_ids, attention_mask=attention_mask)
 last_hidden_states = moutput[0]
else:
moutput = self.model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 last_hidden_states = moutput[0]
last_hidden_states = torch.mean(last_hidden_states, 1)
 X = self.dropout(self.nonlin(self.batchnorm(self.dense(last_hidden_states))))
out_logits = self.outDense(X)
 if labels is None:
Logits = namedtuple('Logits',['logits'])
 out_logits = Logits(out_logits)
 return out_logits
log_ps = self.outActivtn(out_logits)
 batchLoss = self.NLLLoss(log_ps, labels)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_logits)
 return loss_logits<load_pretrained>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embed_glove = load_embed(glove )
model = MyGAPModelForSeqClf(bert_model_name)
 try:
model.load_state_dict(torch.load(".. /input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin"))
except:
model.load_state_dict(torch.load(".. /input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin", map_location='cpu'))
warnings.filterwarnings("ignore")
 model.cuda()<choose_model_class>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
training_args = TrainingArguments(
 output_dir='./results',
num_train_epochs=1,
warmup_steps=1000,
eval_steps=5000,
max_steps=5000,
learning_rate=5e-4,
per_device_train_batch_size=16,
per_device_eval_batch_size=16, 
)
 trainer = MyTrainer(
 model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
tokenizer=tokenizer,
compute_metrics=compute_metrics, 
)
<choose_model_class>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Se encontraron embeddings para {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
training_args = TrainingArguments(
 output_dir='./results',
num_train_epochs=1,
warmup_steps=1000,
eval_steps=5000,
learning_rate=1e-5,
per_device_train_batch_size=24,
per_device_eval_batch_size=24, 
)
 trainer = MyTrainer(
 model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
tokenizer=tokenizer,
compute_metrics=compute_metrics, 
)
 training_stats = trainer.train()<predict_on_test>
combined_df = pd.concat([train_df ,test_df])
 vocab = build_vocab(combined_df['question_text'])
predictions, true_labels, metrics_dummy = trainer.predict(test_dataset)
 pred_labels = np.argmax(predictions, axis=1)
 pred_labels<save_to_csv>
combined_df['question_text'] = combined_df['question_text'].apply(lambda x: x.lower())
 def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print("Anadidas {count} palabras al embedding")
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove)
 add_lower(embed_glove, vocab)
 oov_glove = check_coverage(vocab, embed_glove )
submitDF = original_test_df[['id']]
submitDF['prediction'] = pred_labels
submitDF.prediction = submitDF.prediction.astype(int)
 submitDF.to_csv('submission.csv', index=False)
 submitDF.head()<install_modules>
contraction_mapping = {"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have","so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have"}
!pip install transformers<load_pretrained>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 tokenizer = AutoTokenizer.from_pretrained("joeddav/xlm-roberta-large-xnli", padding=True)
 model = AutoModelForSequenceClassification.from_pretrained("joeddav/xlm-roberta-large-xnli" ).to(device)
 model.config<load_from_csv>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
train_data = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv', encoding='utf8')
 test_data = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv', encoding='utf8')
 train_data.head()<feature_engineering>
combined_df['question_text'] = combined_df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
train_data['label'] = train_data['label'].replace([0, 2], [2, 0])
 train_data.head()<feature_engineering>
def unknown_punct(embed, carc_esp):
unknown = ''
for c in carc_esp:
if c not in embed:
unknown += c
unknown += ' '
return unknown
class MyDataset(Dataset):
def __init__(self, df, tokenizer, labels = False):
self.inputs = df.loc[:,['premise', 'hypothesis']].values
self.tokenizer = tokenizer
self.labels = labels
if self.labels:
self.tgt = df['label'].values
def __len__(self):
return len(self.inputs)
 def __getitem__(self, idx):
inputs = tokenizer(self.inputs[idx].tolist() , add_special_tokens=True, padding=True, return_tensors='pt')
 if self.labels:
inputs['labels'] = self.tgt[idx]
return inputs
return inputs<train_model>
carc_esp_mapping = {"‘": "'",
"₹": "e",
"´": "'",
"°": "",
"€": "e",
"™": "tm",
"√": " sqrt ",
"×": "x",
"²": "2",
"—": "-",
"–": "-",
"’": "'",
"_": "-",
"`": "'",
'“': '"',
'”': '"',
'“': '"',
"£": "e",
'∞': 'infinity',
'θ': 'theta',
'÷': '/',
'α': 'alpha',
'•': '.',
'à': 'a',
'−': '-',
'β': 'beta',
'∅': '',
'³': '3',
'π': 'pi'}
def eval_model(model, dataloader, device):
correct = 0
eval_loss = 0
model.eval()
for i, batch in enumerate(dataloader):
out = model(input_ids=batch['input_ids'].squeeze().to(device),
attention_mask=batch['attention_mask'].squeeze().to(device),
labels=batch['labels'].squeeze().to(device))
eval_loss += out[0].item()
correct +=(out[1].argmax(dim=1)==batch['labels'].squeeze().to(device)).float().sum()
accu = correct / train_dataloader.dataset.__len__()
eval_loss /= len(dataloader)
 return eval_loss, accu<choose_model_class>
def clean_special_chars(text, carc_esp, mapping):
for c in mapping:
text = text.replace(c, mapping[c])
 for c in carc_esp:
text = text.replace(c, f' {c} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
train_dataset = MyDataset(train_data.loc[0:1000], tokenizer, labels=True)
 train_dataloader = DataLoader(dataset=train_dataset,
sampler=BatchSampler(
 SequentialSampler(train_dataset),
batch_size=8, drop_last=True))
loss, accuracy = eval_model(model, train_dataloader, device)
 print("Loss: {}, Accuracy: {}".format(loss, accuracy))<predict_on_test>
combined_df['question_text'] = combined_df['question_text'].apply(lambda x: clean_special_chars(x, carc_esp, carc_esp_mapping))
vocab = build_vocab(combined_df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
def submission_predict(model, dataloader, device):
model.eval()
predicts = np.array([])
 for i, batch in enumerate(dataloader):
inp_ids = batch['input_ids'].squeeze().to(device)
 mask = batch['attention_mask'].squeeze().to(device)
 out = model(input_ids=inp_ids, attention_mask=mask)
 batch_preds = out[0].argmax(dim=1)
 predicts = np.concatenate(( predicts, batch_preds.cpu().detach().numpy()))
return predicts<predict_on_test>
mispell_dict = {'advanatges': 'advantages',
'irrationaol': 'irrational' ,
'defferences': 'differences',
'lamboghini':'lamborghini',
'hypothical':'hypothetical',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'pokémon': 'pokemon'}
test_dataset = MyDataset(test_data, tokenizer, labels=False)
 test_dataloader = DataLoader(dataset=test_dataset,
sampler=BatchSampler(
 SequentialSampler(test_dataset),
batch_size=8, drop_last=False), shuffle=False)
 test_preds = submission_predict(model, test_dataloader, device )<save_to_csv>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
submission = np.concatenate(( test_data['id'].values.reshape(-1,1), np.int32(test_preds.reshape(-1,1))), axis=1)
 submission = pd.DataFrame(submission, columns=['id', 'prediction'])
 submission['prediction'] = submission['prediction'].astype(np.int32 ).replace([0,2], [2,0])
 submission.to_csv('submission.csv', index=False)
 model.save_pretrained('./model.pt')
 tokenizer.save_pretrained('.. /tokenizer' )<install_modules>
combined_df['question_text'] = combined_df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
vocab = build_vocab(combined_df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
!pip install googletrans textAugment<define_search_model>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', ' number ', x)
 x = re.sub('[0-9]{4}', ' number ', x)
 x = re.sub('[0-9]{3}', ' number ', x)
 x = re.sub('[0-9]{2}', ' number ', x)
 return x
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
combined_df['question_text'] = combined_df['question_text'].apply(lambda x: clean_numbers(x))
vocab = build_vocab(combined_df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
df_train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv' )<import_modules>
train_df['treated_question'] = train_df['question_text'].apply(lambda x: x.lower())
 train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_special_chars(x, carc_esp, carc_esp_mapping))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_numbers(x))
from textaugment import EDA
from googletrans import Translator<count_unique_values>
test_df['treated_question'] = test_df['question_text'].apply(lambda x: x.lower())
 test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_special_chars(x, carc_esp, carc_esp_mapping))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_numbers(x))
for lang in df_train['language'].unique() :
number_of_training_samples = df_train[df_train['language'] == lang].shape[0] / df_train.shape[0]
number_of_testing_samples = df_test[df_test['language'] == lang].shape[0] / df_test.shape[0]
print('distribution of {} in training samples: {} and in testing samples {}'.format(lang, number_of_training_samples, number_of_testing_samples))<count_unique_values>
def load_glove_matrix(word_index, embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
idx2label = {0: 'entailment', 1 :'neutral', 2: 'contradiction'}
for label in df_train['label'].unique() :
number_of_training_samples = df_train[df_train['label'] == label].shape[0] / df_train.shape[0]
print('distribution of {} in training samples: {}'.format(idx2label[label], number_of_training_samples)) 
<set_options>
np.random.seed(2)
 train_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[train_idx]
train_y = train_y[train_idx]
val_X = val_X[val_idx]
val_y = val_y[val_idx]
embedding_matrix_glove = load_glove_matrix(tokenizer.word_index, embed_glove )
tf.random.set_seed(seed )<load_pretrained>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
model_name = 'jplu/tf-xlm-roberta-large'
tokenizer = XLMRobertaTokenizer.from_pretrained(model_name )<feature_engineering>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def clean_word(value):
language = value[0]
word = value[1]
if language != 'English':
word = word.lower()
return word
word = word.lower()
word = re.sub(r'\?\?', 'e', word)
 word = re.sub('\.\.\.', '.', word)
 word = re.sub('\/', ' ', word)
 word = re.sub('--', ' ', word)
 word = re.sub('/\xad', '', word)
 word = word.strip(' ')
 return word
df_train['premise'] = df_train[['language', 'premise']].apply(lambda v: clean_word(v), axis=1)
 df_train['hypothesis'] = df_train[['language', 'hypothesis']].apply(lambda v: clean_word(v), axis=1)
 df_test['premise'] = df_test[['language', 'premise']].apply(lambda v: clean_word(v), axis=1)
 df_test['hypothesis'] = df_test[['language', 'hypothesis']].apply(lambda v: clean_word(v), axis=1 )<categorify>
def model_lstm_att(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 attn = Attention(maxlen )(x)
 y = Dense(32, activation='relu' )(attn)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy', 'accuracy'])
 return model
def build_model() :
with strategy.scope() :
bert_encoder = TFXLMRobertaModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape=(None,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(None,), dtype=tf.int32, name="input_mask")
 embedding = bert_encoder([input_word_ids, input_mask])[0]
output_layer = tf.keras.layers.Dropout(0.25 )(embedding)
 output_layer = tf.keras.layers.GlobalAveragePooling1D()(output_layer)
 output_dense_layer = tf.keras.layers.Dense(64, activation='relu' )(output_layer)
 output_dense_layer = tf.keras.layers.Dense(32, activation='relu' )(output_dense_layer)
 output = tf.keras.layers.Dense(3, activation='softmax' )(output_dense_layer)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 return model<define_variables>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val puntuacion F1: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 return pred_val_y, pred_test_y, best_score
batch_size = 8 * strategy.num_replicas_in_sync
num_splits = 5
test_input = None<create_dataframe>
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )
auto = tf.data.experimental.AUTOTUNE
languages = [ 'zh-cn' if lang == 'zh' else lang for lang in df_train['lang_abv'].unique() ]
def make_dataset(train_input, train_label):
dataset = tf.data.Dataset.from_tensor_slices(
(
 train_input,
train_label 
)
 ).repeat().shuffle(batch_size ).batch(batch_size ).prefetch(auto)
 return dataset<categorify>
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("La puntuación F1 en los limites {0} y {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
def xlm_roberta_encode(hypotheses, premises, src_langs, augmentation=False):
num_examples = len(hypotheses)
 sentence_1 = [tokenizer.encode(s)for s in premises]
sentence_2 = [tokenizer.encode(s)for s in hypotheses]
input_word_ids = list(map(lambda x: x[0]+x[1], list(zip(sentence_1,sentence_2))))
input_mask = [np.ones_like(x)for x in input_word_ids]
inputs = {
'input_word_ids': tf.keras.preprocessing.sequence.pad_sequences(input_word_ids, padding='post'),
'input_mask': tf.keras.preprocessing.sequence.pad_sequences(input_mask, padding='post')
 }
return inputs<split>
print("Mejor limite:", best_thresh, "y puntuacion F1 ", thresholds[0][1] )
train_df, validation_df = train_test_split(df_train, test_size=0.1)
 if test_input is None:
test_input = xlm_roberta_encode(df_test.hypothesis.values, df_test.premise.values, df_test.lang_abv.values,augmentation=False)
 df_train['prediction'] = 0
num_augmentation = 1
train_input = xlm_roberta_encode(train_df.hypothesis.values,train_df.premise.values, train_df.lang_abv.values, augmentation=False)
 train_label = train_df.label.values
train_sequence = make_dataset(train_input, train_label)
 n_steps =(len(train_label)) // batch_size
validation_input = xlm_roberta_encode(validation_df.hypothesis.values, validation_df.premise.values,validation_df.lang_abv.values, augmentation=False)
 validation_label = validation_df.label.values
tf.keras.backend.clear_session()
with strategy.scope() :
model = build_model()
history = model.fit(
 train_sequence, shuffle=True, steps_per_epoch=n_steps,
validation_data =(validation_input, validation_label), epochs=50, verbose=1,
callbacks=[
tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10),
tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=5),
tf.keras.callbacks.ModelCheckpoint(
 'model.h5', monitor='val_accuracy', save_best_only=True,save_weights_only=True)
 ] 
)
 model.load_weights('model.h5')
 validation_predictions = model.predict(validation_input)
 validation_predictions = np.argmax(validation_predictions, axis=-1)
 validation_df['predictions'] = validation_predictions
acc = accuracy_score(validation_label, validation_predictions)
 print('Accuracy: {}'.format(acc))
test_split_predictions = model.predict(test_input)
 del train_input, train_label, validation_input, validation_label, model, train_sequence
gc.collect()<compute_test_metric>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )
fig, axes = plt.subplots(nrows=4, ncols=4, figsize=(15,10))
labels = list(idx2label.values())
 for ax,language in zip(axes.flatten() [:validation_df['language'].nunique() ],validation_df['language'].unique()):
y_pred = validation_df[validation_df['language'] == language]['prediction'].values
y_true = validation_df[validation_df['language'] == language]['label'].values
lang_acc = accuracy_score(y_true, y_pred)
 print('Language {} has accuracy {}'.format(language, lang_acc))
cm = confusion_matrix(np.array([idx2label[v] for v in y_true]),
np.array([idx2label[v] for v in y_pred]),
labels=labels)
 cax = ConfusionMatrixDisplay(cm, display_labels=labels)
 cax.plot(ax=ax)
 ax.set_title(language)
 ax.set_xticklabels([''] + labels)
 ax.set_yticklabels([''] + labels)
 plt.title('Confusion matrix of the classifier')
 plt.xlabel('Predicted')
 plt.ylabel('True')
 plt.tight_layout()
plt.show()<create_dataframe>
precisiones_globales=[]
def precision(model_lstm, registrar=False):
y_pred1 = model_lstm.predict(train_X)
 train_auc1 = roc_auc_score(train_y, y_pred1)
 y_pred1 = model_lstm.predict(val_X)
 val_auc1 = roc_auc_score(val_y, y_pred1)
 print('Train AUC: ', train_auc1)
 print('Vali AUC: ', val_auc1)
 if registrar:
precisiones_globales.append([train_auc1,val_auc1] )
predictions = np.argmax(test_split_predictions, axis=-1)
 submission = df_test.id.copy().to_frame()
submission['prediction'] = predictions<save_to_csv>
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )