Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<import_modules>
system('pip install pycaret');
import numpy as np
import torch
from torch.utils.data import DataLoader, Dataset, TensorDataset
import sys
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
import torch.optim as optim
import seaborn as sns
from collections import defaultdict
import time
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup<set_options>
from os import makedirs
import pandas as pd
from pycaret.regression import *
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print(device )<load_from_csv>
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv', index_col='id')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv', index_col='id' )
train_csv = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv', keep_default_na = False)
 train_csv = train_csv.sample(frac=1 ).reset_index(drop=True)
 ninetyfive_percent = round(0.90*(len(train_csv)))
 train_data = train_csv.iloc[:ninetyfive_percent]
valid_data = train_csv.iloc[ninetyfive_percent:]
print('Number of Training samples: ', len(train_data))
print('Number of Validation samples: ',len(valid_data))<load_from_csv>
setup(data=train,
target='target',
fold=5,
silent=True,
verbose=False,
session_id=2021);
test_csv = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv', keep_default_na = False)
 test_dataset = mydataset(test_csv , name = 'test')
 test_dataloader = data.DataLoader(test_dataset, shuffle= False, batch_size = 1, num_workers=16,pin_memory=True )<train_model>
xgb = create_model('xgboost' )
def train(model, data_loader, valid_loader, criterion, optimizer, lr_scheduler, modelpath, device, epochs):
model.train()
train_loss= []
valid_loss = []
valid_acc = []
for epoch in range(epochs):
avg_loss = 0.0
for batch_num,(tweet, input_id, attention_masks, target)in enumerate(data_loader):
input_ids, attention_masks, target = input_id.to(device), attention_masks.to(device), target.to(device)

loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
labels=target,
return_dict=False 
)

optimizer.zero_grad()
loss.backward()
optimizer.step()

 torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 avg_loss += loss.item()

 lr_scheduler.step()
training_loss = avg_loss/len(data_loader)
 print('Epoch: ', epoch+1)
 print('training loss = ', training_loss)
 train_loss.append(training_loss)

validation_loss, top1_acc= test_classify(model, valid_loader, criterion, device)
 valid_loss.append(validation_loss)
 valid_acc.append(top1_acc)

torch.save({
'model_state_dict': model.state_dict() ,
'optimizer_state_dict': optimizer.state_dict() ,
'lr_scheduler': lr_scheduler.state_dict() ,
}, modelpath)
 return train_loss, valid_loss, valid_acc

 def test_classify(model, valid_loader, criterion, device):
model.eval()
test_loss = []
top1_accuracy = 0
total = 0
for batch_num,(tweet, input_id, attention_masks, target)in enumerate(valid_loader):
input_ids, attention_masks, target = input_id.to(device), attention_masks.to(device), target.to(device)

loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
labels=target,
return_dict=False)
 test_loss.extend([loss.item() ]*input_id.size() [0])
 predictions = F.softmax(logits, dim=1)
 _, top1_pred_labels = torch.max(predictions,1)
 top1_pred_labels = top1_pred_labels.view(-1)
 top1_accuracy += torch.sum(torch.eq(top1_pred_labels, target)).item()
total += len(target)
 print('Validation Loss: {:.4f}\tTop 1 Validation Accuracy: {:.4f}'.format(np.mean(test_loss), top1_accuracy/total))
return np.mean(test_loss), top1_accuracy/total<train_model>
tuned_xgb = tune_model(xgb,
custom_grid=None,
optimize='RMSE',
search_library='scikit-optimize',
n_iter=50,
choose_better=True )
modelname = 'BERT'
modelpath = 'saved_checkpoint_'+modelname
train_loss, valid_loss, valid_acc = train(model, train_dataloader, validation_dataloader, criterion, optimizer, lr_scheduler, modelpath, device, epochs = num_Epochs )<predict_on_test>
lgb = create_model('lightgbm' )
def predict(model, test_loader, device):
model.eval()
target = []
for batch_num,(captions, input_id, attention_masks)in enumerate(test_loader):
input_ids, attention_masks = input_id.to(device), attention_masks.to(device)

output_dictionary = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
return_dict=True)
 predictions = F.softmax(output_dictionary['logits'], dim=1)
 _, top1_pred_labels = torch.max(predictions,1)
 top1_pred_labels = top1_pred_labels.item()
target.append(top1_pred_labels)
 make_csv(target)
 def make_csv(target):
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 my_submission = pd.DataFrame({'id': test.id, 'target': target})
 my_submission.to_csv('submission.csv', index=False)
<predict_on_test>
tuned_lgb = tune_model(lgb,
custom_grid=None,
optimize='RMSE',
search_library='scikit-optimize',
n_iter=50,
choose_better=True )
predict(model, test_dataloader, device )<install_modules>
cat = create_model('catboost' )
!pip install pytorch-pretrained-bert pytorch-nlp<set_options>
tuned_cat = tune_model(cat,
custom_grid=None,
optimize='RMSE',
search_library='scikit-optimize',
n_iter=50,
choose_better=True )
warnings.filterwarnings('ignore')
<load_pretrained>
stacked_model = stack_models(estimator_list=[tuned_xgb, tuned_lgb, cat] )
nltk.download('punkt' )<set_options>
best_model = automl(optimize='RMSE')
 best_model
pd.options.display.max_colwidth = 100
seed_val=42
tf.random.set_seed(seed_val)
 random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )<set_options>
final_xgb = finalize_model(tuned_xgb)
 final_lgb = finalize_model(tuned_lgb)
 final_cat = finalize_model(cat)
 final_stack = finalize_model(stacked_model)
 final_best = finalize_model(best_model )
<load_from_csv><EOS> 
names = ['xgb', 'lgb', 'cat', 'stack', 'best']
models = [final_xgb, final_lgb, final_cat, final_stack, final_best]
for name, model in zip(names, models):
try:
prediction = predict_model(model, test)
 prediction['Label'].rename('target' ).to_csv(f'submission_{name}.csv')
 except:
print(f'error:{name}' )