Kaggle competition: Contradictory, My Dear Watson
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.describe()<drop_column>
!pip install datasets
!pip install --upgrade seaborn
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )<drop_column>
import os
import seaborn as sns
import numpy as np
import random
from sklearn.utils import shuffle
import matplotlib.pyplot as plt
import pandas as pd
from collections import namedtuple
import plotly.express as px
import transformers
import tokenizers
import datetime
import time
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef
import tensorflow
from datasets import load_dataset
import torch
from torch import nn
from transformers import BertTokenizer, AutoTokenizer, TrainingArguments, AutoModel, AutoModelForSequenceClassification
from torch.utils.data import TensorDataset, random_split
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup
from transformers import RobertaTokenizer, RobertaForSequenceClassification, XLMRobertaModel
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )<feature_engineering>
mnli = load_dataset('glue', 'mnli')
 len(mnli )
train["SalePrice"] = np.log1p(train["SalePrice"])
 check_skewness('SalePrice' )<prepare_x_and_y>
xnli = load_dataset('xnli', 'all_languages')
 len(xnli )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<create_dataframe>
original_train_df = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train_df = shuffle(original_train_df)
 print(original_train_df.shape)
 original_valid_df = original_train_df[:len(original_train_df)//2]
original_train_df = original_train_df[(len(original_train_df)//2):]
print(original_train_df.shape, original_valid_df.shape)
 original_train_df.head()
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na} )<filter>
original_test_df = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 original_test_df.shape
all_data.PoolQC.loc[all_data.PoolQC.notnull() ]<data_type_conversions>
bert_model_name = 'joeddav/xlm-roberta-large-xnli'
max_len = 128
tokenizer = AutoTokenizer.from_pretrained(bert_model_name )
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )<data_type_conversions>
def encode(premise, hypothesis, label):
encoded_dict = tokenizer(
 premise,
hypothesis,
add_special_tokens = True,
truncation=True,
max_length = max_len,
padding='max_length',
return_attention_mask = True,
return_tensors = 'pt', 
)
 for k in encoded_dict:
encoded_dict[k] = torch.squeeze(encoded_dict[k])
 encoded_dict['labels'] = label
return encoded_dict
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )<data_type_conversions>
raw_ds_mapping = {
'original train':(_get_raw_datasets_from_dataframe, original_train_df, len(original_train_df)) ,
'original valid':(_get_raw_datasets_from_dataframe, original_valid_df, len(original_valid_df)) ,
'mnli train' :(_get_raw_datasets_from_nlp, mnli['train'], mnli['train'].num_rows),
'mnli valid 1' :(_get_raw_datasets_from_nlp, mnli['validation_matched'], mnli['validation_matched'].num_rows),
'mnli valid 2' :(_get_raw_datasets_from_nlp, mnli['validation_mismatched'], mnli['validation_mismatched'].num_rows),
'xnli valid' :(_get_raw_datasets_from_nlp, xnli['validation'], xnli['validation'].num_rows * 15),
'xnli test' :(_get_raw_datasets_from_nlp, xnli['test'], xnli['test'].num_rows * 15),
'original test' :(_get_raw_datasets_from_dataframe, original_test_df, len(original_test_df)) ,
}
def get_raw_dataset(ds_name):
fn, ds, nb_examples = raw_ds_mapping[ds_name]
for x in fn(ds):
yield x
return
all_data["Alley"] = all_data["Alley"].fillna("None" )<data_type_conversions>
train_ds_names = ['original train', 'mnli train', 'mnli valid 1', 'mnli valid 2', 'xnli valid', 'xnli test']
eval_ds_names = ['original valid']
test_ds_names = ['original test']
all_data["Fence"] = all_data["Fence"].fillna("None" )<data_type_conversions>
class MyIterableDataset(torch.utils.data.IterableDataset):
def __init__(self, ds_names, isLabeledDataset=False):
super(MyIterableDataset ).__init__()
self.ds_names = ds_names
self.length = self.getLen()
self.isLabeledDataset = isLabeledDataset
return
def getLen(self):
length = 0
for ds_name in self.ds_names:
fn, ds, nb_examples = raw_ds_mapping[ds_name]
length += nb_examples
return length
def __iter__(self):
for ds_name in self.ds_names:
fn, ds, nb_examples = raw_ds_mapping[ds_name]
for x in fn(ds):
if self.isLabeledDataset and x['labels'] == -1:
continue
yield x
return
def __len__(self):
return self.length
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )<groupby>
train_dataset = MyIterableDataset(train_ds_names, True)
 for i, d in enumerate(train_dataset):
if i==0:
print(d)
 break
grouped_df = all_data.groupby('Neighborhood')['LotFrontage']
for key, item in grouped_df:
print(key,"
")
 print(grouped_df.get_group(key))
break<categorify>
def get_dataLoaders(train_dataset, per_device_train_batch_size=64, epochs=1):
if isinstance(train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = SequentialSampler(train_dataset)
 train_dataloader = DataLoader(
 train_dataset,
sampler = train_sampler,
batch_size = per_device_train_batch_size 
)
 return train_dataloader
def test_iterate_dataloader(train_dataloader):
for step, batch in enumerate(train_dataloader):
print(step, batch)
 return
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))<data_type_conversions>
class My_Dataset(torch.utils.data.Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
return
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx])for key, val in self.encodings.items() }
if self.labels is not None and len(self.labels)>0:
item['labels'] = self.labels[idx]
return item
def __len__(self):
return len(self.labels )
for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
all_data[col] = all_data[col].fillna('None' )<groupby>
def get_mapType_Dataset(ds_names, isLabeledDataset):
input_ids = []
attention_mask = []
labels = []
lang = []
dataset_iter = MyIterableDataset(ds_names, isLabeledDataset)
 for encoded_dict in dataset_iter:
input_ids.append(encoded_dict['input_ids'])
 attention_mask.append(encoded_dict['attention_mask'])
 labels.append(encoded_dict['labels'])
 encodings = {'input_ids':input_ids, 'attention_mask':attention_mask}
my_dataset = My_Dataset(encodings, labels)
 return my_dataset
abc = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','GarageYrBlt', 'GarageArea', 'GarageCars']
all_data.groupby('GarageType')[abc].count()<data_type_conversions>
eval_dataset = get_mapType_Dataset(eval_ds_names, True)
 test_dataset = get_mapType_Dataset(test_ds_names, False )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
def save_my_model_and_tokenizer(model, tokenizer=None, output_dir='./model_save/'):

 model_state_file = output_dir + 'my_model.bin'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 print("Saving model to %s" % model_state_file)
 model.cpu()
state = model.state_dict()
torch.save(state, model_state_file)
 if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
 model.cuda()
return
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
def load_model(bert_model_name, output_dir='./model_save/'):
model_state_file = output_dir + 'my_model.bin'
model = MyGAPModelForSeqClf(bert_model_name)
 state = torch.load(model_state_file)
 model.load_state_dict(state)
 return model
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<count_values>
def compute_metrics(labels, pred_logits):
preds = pred_logits.argmax(-1)
 precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
 acc = accuracy_score(labels, preds)
 mcc = matthews_corrcoef(labels, preds)
 metrics = {
'mcc': mcc,
'accuracy': acc,
'f1': f1,
'precision': precision,
'recall': recall
}
return metrics
all_data['MSZoning'].value_counts()<data_type_conversions>
class MyTrainer:
def __init__(self, model, args, train_dataset, eval_dataset, tokenizer, compute_metrics=compute_metrics):
self.model = model
self.args = args
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.tokenizer = tokenizer
self.compute_metrics = compute_metrics
self.isTrained = False
self.device = self.get_device_type()
self.optimizer = AdamW(model.parameters() ,
lr = args.learning_rate,
eps = args.adam_epsilon 
)
 self.epochs = self.args.num_train_epochs
self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders()
return
def get_device_type(self):
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 return device
def get_dataLoaders(self):
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = SequentialSampler(self.train_dataset)
 train_dataloader = DataLoader(
 self.train_dataset,
sampler = train_sampler,
batch_size = self.args.per_device_train_batch_size 
)
 validation_dataloader = DataLoader(
 self.eval_dataset,
sampler = SequentialSampler(self.eval_dataset),
batch_size = self.args.per_device_eval_batch_size 
)
 num_training_steps = len(train_dataloader)* self.epochs
lr_scheduler = get_linear_schedule_with_warmup(self.optimizer,
num_warmup_steps = self.args.warmup_steps,
num_training_steps = num_training_steps)
 return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps
def test_iterate_dataloader() :
for step, batch in enumerate(self.train_dataloader):
print(step, batch)
 return
def train(self, is_train_base_encoding_model=True):
if is_train_base_encoding_model:
self.model.unfreez()
else:
self.model.freez()
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
min_val_loss = 99999999
for epoch_i in range(0, self.epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, self.epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
self.model.train()
for step, batch in enumerate(self.train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(self.train_dataloader), elapsed))
if(self.args.max_steps > 0 and self.args.max_steps < step)or \ 
(is_train_base_encoding_model and self.args.eval_steps>0 and step % self.args.eval_steps==0 and step>0):
avg_train_loss = total_train_loss / step
training_time = format_time(time.time() - t0)
 print("Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
save_my_model_and_tokenizer(self.model, self.tokenizer, output_dir='./model_save/')
 if self.args.max_steps > 0 and self.args.max_steps < step:
return training_stats
self.model.zero_grad()
for k in batch:
batch[k] = batch[k].to(self.device)
 output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters() , 1.0)
 self.optimizer.step()
self.lr_scheduler.step()
avg_train_loss = total_train_loss / len(self.train_dataloader)
 training_time = format_time(time.time() - t0)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("
Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
save_my_model_and_tokenizer(self.model, self.tokenizer, output_dir='./model_save/')
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.plot_train_stats(training_stats)
 return training_stats
def evaluate(self):
t0 = time.time()
self.model.eval()
total_eval_mcc = 0
total_eval_f1 = 0
total_eval_precision = 0
total_eval_recall = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in self.validation_dataloader:
with torch.no_grad() :
for k in batch:
batch[k] = batch[k].to(self.device)
 output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = batch['labels'].to('cpu' ).numpy()
metrics = self.compute_metrics(label_ids, logits)
 total_eval_mcc += metrics['mcc']
total_eval_f1 += metrics['f1']
total_eval_precision += metrics['precision']
total_eval_recall += metrics['recall']
total_eval_accuracy += metrics['accuracy']
avg_val_f1 = total_eval_f1 / len(self.validation_dataloader)
 print(" F1: {0:.3f}".format(avg_val_f1))
avg_val_mcc = total_eval_mcc / len(self.validation_dataloader)
 print(" MCC: {0:.3f}".format(avg_val_mcc))
avg_val_precision = total_eval_precision / len(self.validation_dataloader)
 print(" Precision: {0:.3f}".format(avg_val_precision))
avg_val_recall = total_eval_recall / len(self.validation_dataloader)
 print(" Recall: {0:.3f}".format(avg_val_recall))
avg_val_accuracy = total_eval_accuracy / len(self.validation_dataloader)
 print(" Accuracy: {0:.3f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(self.validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time
def plot_train_stats(self, training_stats):
mccs = [e['eval_mcc'] for e in training_stats]
accuracies = [e['eval_accuracy'] for e in training_stats]
f1_scores = [e['eval_f1'] for e in training_stats]
precisions = [e['eval_precision'] for e in training_stats]
recalls = [e['eval_recall'] for e in training_stats]
losses = [e['eval_loss'] for e in training_stats]
epochs = training_stats[-1]['epoch']
print('mccs:', mccs)
 print('accuracies:', accuracies)
 print('precisions:', precisions)
 print('recalls:', recalls)
 print('f1_scores:', f1_scores)
 print('losses:', losses)
 sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_mcc')
 sns.lineplot(x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')
 sns.lineplot(x=np.arange(1, epochs + 1), y=precisions, label='val_precision')
 sns.lineplot(x=np.arange(1, epochs + 1), y=recalls, label='val_recall')
 sns.lineplot(x=np.arange(1, epochs + 1), y=f1_scores, label='val_f1')
 plt.show()
return
def getTrainedModel(self):
if self.isTrained:
return self.model
return None
def predict(self, prediction_dataset, isRemoveLabels=True):

 prediction_sampler = SequentialSampler(prediction_dataset)
 prediction_dataloader = DataLoader(prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)
 print('Predicting labels for {:,} test sentences...'.format(len(prediction_dataset)))
 self.model.eval()
predictions = []
for batch in prediction_dataloader:
batch = {t:batch[t].to(self.device)for t in batch}
with torch.no_grad() :
if isRemoveLabels:
batch.pop('labels')
 for k in batch:
batch[k] = batch[k].to(self.device)
 outputs = model(**batch)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')
 print('Done prediction')
 pred_logits = np.concatenate(predictions, axis=0)
 return pred_logits, None, None
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<count_values>
class MyGAPModelForSeqClf(nn.Module):
def __init__(self, bert_model_name, outputCount=3, drop_prob=0.1, nonlin=nn.ReLU()):
super(MyGAPModelForSeqClf, self ).__init__()
self.model = AutoModelForSequenceClassification.from_pretrained(bert_model_name ).base_model
self.drop_prob = drop_prob
self.nonlin = nonlin
self.outputCount = outputCount
hidden_size = self.model.config.hidden_size
self.dense = nn.Linear(hidden_size, hidden_size)
 self.batchnorm = nn.BatchNorm1d(hidden_size)
 self.outDense = nn.Linear(hidden_size, outputCount)
 self.dropout = nn.Dropout(drop_prob)
 self.outActivtn = nn.LogSoftmax(dim=1)
 self.NLLLoss = nn.NLLLoss()
return
def freez(self):
for param in self.model.base_model.parameters() :
param.requires_grad = False
return
def unfreez(self):
for param in self.model.base_model.parameters() :
param.requires_grad = True
return
def forward(self, input_ids, attention_mask, token_type_ids=None, labels=None, **kwargs):
last_hidden_states = None
if token_type_ids is None:
moutput = self.model(input_ids=input_ids, attention_mask=attention_mask)
 last_hidden_states = moutput[0]
else:
moutput = self.model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 last_hidden_states = moutput[0]
last_hidden_states = torch.mean(last_hidden_states, 1)
 X = self.dropout(self.nonlin(self.batchnorm(self.dense(last_hidden_states))))
out_logits = self.outDense(X)
 if labels is None:
Logits = namedtuple('Logits',['logits'])
 out_logits = Logits(out_logits)
 return out_logits
log_ps = self.outActivtn(out_logits)
 batchLoss = self.NLLLoss(log_ps, labels)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_logits)
 return loss_logits
all_data['Utilities'].value_counts()<drop_column>
model = MyGAPModelForSeqClf(bert_model_name)
 try:
model.load_state_dict(torch.load(".. /input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin"))
except:
model.load_state_dict(torch.load(".. /input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin", map_location='cpu'))
warnings.filterwarnings("ignore")
 model.cuda()
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
training_args = TrainingArguments(
 output_dir='./results',
num_train_epochs=1,
warmup_steps=1000,
eval_steps=5000,
max_steps=5000,
learning_rate=5e-4,
per_device_train_batch_size=16,
per_device_eval_batch_size=16, 
)
 trainer = MyTrainer(
 model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
tokenizer=tokenizer,
compute_metrics=compute_metrics, 
)
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<categorify>
training_args = TrainingArguments(
 output_dir='./results',
num_train_epochs=1,
warmup_steps=1000,
eval_steps=5000,
learning_rate=1e-5,
per_device_train_batch_size=24,
per_device_eval_batch_size=24, 
)
 trainer = MyTrainer(
 model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
tokenizer=tokenizer,
compute_metrics=compute_metrics, 
)
 training_stats = trainer.train()
mode_col = ['Electrical','KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType']
for col in mode_col:
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )<data_type_conversions>
predictions, true_labels, metrics_dummy = trainer.predict(test_dataset)
 pred_labels = np.argmax(predictions, axis=1)
 pred_labels
<sort_values><EOS> 
submitDF = original_test_df[['id']]
submitDF['prediction'] = pred_labels
submitDF.prediction = submitDF.prediction.astype(int)
 submitDF.to_csv('submission.csv', index=False)
 submitDF.head()