Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<create_dataframe>
import numpy as np
import torch
from torch.utils.data import DataLoader, Dataset, TensorDataset
import sys
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
import torch.optim as optim
import seaborn as sns
from collections import defaultdict
import time
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True,
)<load_pretrained>
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print(device )
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<choose_model_class>

 train_csv = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv', keep_default_na = False)
 train_csv = train_csv.sample(frac=1 ).reset_index(drop=True)
 ninetyfive_percent = round(0.90*(len(train_csv)))
 train_data = train_csv.iloc[:ninetyfive_percent]
valid_data = train_csv.iloc[ninetyfive_percent:]
print('Number of Training samples: ', len(train_data))
print('Number of Validation samples: ',len(valid_data))
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load('plabel_model/last-checkpoint1.bin')
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
return DetBenchTrain(net, config)
 net = get_net()<train_model>

 test_csv = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv', keep_default_na = False)
 test_dataset = mydataset(test_csv , name = 'test')
 test_dataloader = data.DataLoader(test_dataset, shuffle= False, batch_size = 1, num_workers=16,pin_memory=True )
if len(os.listdir('.. /input/global-wheat-detection/test/')) <1:
pass
else:
run_training()<set_options>
def train(model, data_loader, valid_loader, criterion, optimizer, lr_scheduler, modelpath, device, epochs):
model.train()
train_loss= []
valid_loss = []
valid_acc = []
for epoch in range(epochs):
avg_loss = 0.0
for batch_num,(tweet, input_id, attention_masks, target)in enumerate(data_loader):
input_ids, attention_masks, target = input_id.to(device), attention_masks.to(device), target.to(device)

loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
labels=target,
return_dict=False 
)

optimizer.zero_grad()
loss.backward()
optimizer.step()

 torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 avg_loss += loss.item()

 lr_scheduler.step()
training_loss = avg_loss/len(data_loader)
 print('Epoch: ', epoch+1)
 print('training loss = ', training_loss)
 train_loss.append(training_loss)

validation_loss, top1_acc= test_classify(model, valid_loader, criterion, device)
 valid_loss.append(validation_loss)
 valid_acc.append(top1_acc)

torch.save({
'model_state_dict': model.state_dict() ,
'optimizer_state_dict': optimizer.state_dict() ,
'lr_scheduler': lr_scheduler.state_dict() ,
}, modelpath)
 return train_loss, valid_loss, valid_acc

 def test_classify(model, valid_loader, criterion, device):
model.eval()
test_loss = []
top1_accuracy = 0
total = 0
for batch_num,(tweet, input_id, attention_masks, target)in enumerate(valid_loader):
input_ids, attention_masks, target = input_id.to(device), attention_masks.to(device), target.to(device)

loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
labels=target,
return_dict=False)
 test_loss.extend([loss.item() ]*input_id.size() [0])
 predictions = F.softmax(logits, dim=1)
 _, top1_pred_labels = torch.max(predictions,1)
 top1_pred_labels = top1_pred_labels.view(-1)
 top1_accuracy += torch.sum(torch.eq(top1_pred_labels, target)).item()
total += len(target)
 print('Validation Loss: {:.4f}\tTop 1 Validation Accuracy: {:.4f}'.format(np.mean(test_loss), top1_accuracy/total))
return np.mean(test_loss), top1_accuracy/total
time.sleep(1)
 def memory_cleanup() :

 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
torch.cuda.empty_cache()
memory_cleanup()<categorify>
modelname = 'BERT'
modelpath = 'saved_checkpoint_'+modelname
train_loss, valid_loss, valid_acc = train(model, train_dataloader, validation_dataloader, criterion, optimizer, lr_scheduler, modelpath, device, epochs = num_Epochs )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
def predict(model, test_loader, device):
model.eval()
target = []
for batch_num,(captions, input_id, attention_masks)in enumerate(test_loader):
input_ids, attention_masks = input_id.to(device), attention_masks.to(device)

output_dictionary = model(input_ids,
token_type_ids=None,
attention_mask=attention_masks,
return_dict=True)
 predictions = F.softmax(output_dictionary['logits'], dim=1)
 _, top1_pred_labels = torch.max(predictions,1)
 top1_pred_labels = top1_pred_labels.item()
target.append(top1_pred_labels)
 make_csv(target)
 def make_csv(target):
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 my_submission = pd.DataFrame({'id': test.id, 'target': target})
 my_submission.to_csv('submission.csv', index=False)
<load_pretrained><EOS> 
predict(model, test_dataloader, device )