Kaggle competition: Natural Language Processing with Disaster Tweets
num_epochs = 5
batch_size = 32
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed )<load_from_csv>
!pip install transformers
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.Adam(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
import torch
from torch.utils.data import TensorDataset, random_split
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, get_linear_schedule_with_warmup
from transformers import BertForSequenceClassification, AdamW, BertConfig
import torch.nn.functional as F
import pandas as pd
import numpy as np
import time
import datetime
import random, re
import seaborn as sns
import matplotlib.pyplot as plt
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.{}'.format(torch.cuda.device_count()))
print('We will use the GPU: {}'.format(torch.cuda.get_device_name(0)))
 else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 pd.set_option('display.max_colwidth', 150)
 train.head()
warnings.filterwarnings('ignore' )<define_variables>
print('Number of training sentences: {:,}
'.format(train.shape[0]))
print('Number of test sentences: {:,}
'.format(test.shape[0]))
seed=42<feature_engineering>
def clean_text(text):
text = text.lower()
text = re.sub(r'[!]+','!',text)
 text = re.sub(r'[?]+','?',text)
 text = re.sub(r'[.]+','.',text)
 text = re.sub(r"'","",text)
 text = re.sub('\s+', '', text ).strip()
text = re.sub(r'&amp;?',r'and', text)
 text = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", text)
 text = re.sub(r'[:"$%&\*+,-/:;<=>@\\^_`{|}~]+','',text)
 return text
train.head()
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
sentences = train.text.values
labels = train.target.values
sentences_test = test.text.values
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.conv1 = nn.Conv1d(in_channels=768, out_channels=768, kernel_size=3, padding=1)
 self.conv2 = nn.Conv1d(in_channels=768, out_channels=64, kernel_size=3, padding=1)
 self.conv3 = nn.Conv1d(in_channels=768, out_channels=768, kernel_size=3, padding=1)
 self.conv4 = nn.Conv1d(in_channels=768, out_channels=64, kernel_size=3, padding=1)
 self.fc1 = nn.Linear(64, 1)
 self.fc2 = nn.Linear(64, 1)
 nn.init.normal_(self.fc1.weight, std=0.02)
 nn.init.normal_(self.fc1.bias, 0)
 nn.init.normal_(self.fc2.weight, std=0.02)
 nn.init.normal_(self.fc2.bias, 0)
 nn.init.xavier_normal_(self.conv1.weight)
 nn.init.xavier_normal_(self.conv2.weight)
 nn.init.xavier_normal_(self.conv3.weight)
 nn.init.xavier_normal_(self.conv4.weight)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x1=x.permute(0,2,1)
 x1 = self.conv1(x1)
 x1 = self.conv2(F.leaky_relu(x1))
x1 = x1.permute(0,2,1)
 x1 = self.fc1(x1)
 start_logits = x1.squeeze(-1)
 x2 = x.permute(0,2,1)
 x2 = self.conv3(x2)
 x2 = self.conv4(F.leaky_relu(x2))
x2 = x2.permute(0,2,1)
 x2 = self.fc2(x2)
 end_logits = x2.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
print(' Original: ', sentences[0])
 print('Tokenized: ', tokenizer.tokenize(sentences[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
max_len = 0
for sent in sentences:
input_ids = tokenizer.encode(sent, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max sentence length: ', max_len)
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original sentence: ', sentences[0])
 print('Token IDs list:', input_ids[0])
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
if epoch<2:
jaccard_check=epoch_jaccard
torch.save(model.state_dict() , filename)
 print('Saving model')
 elif epoch_jaccard>jaccard_check:
jaccard_check=epoch_jaccard
torch.save(model.state_dict() , filename)
 print('Saving model')
 else:
print('Stopping iteration')
 break<choose_model_class>
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.8 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))
num_epochs = 10
batch_size = 32
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
batch_size = 32
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=4e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8 
)
 epochs = 4
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat)
 def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
warnings.filterwarnings('ignore' )<load_from_csv>
training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took {:} time".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )
train_data = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' )<count_missing_values>
print('Number of test sentences: {:,}
'.format(test.shape[0]))
sentences_test = test.text.values
input_ids = []
attention_masks = []
for sent in sentences_test:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = batch_size
prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )<correct_missing_values>
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
predictions.append(logits)
 print('DONE!')
train_data.dropna(axis = 0,inplace=True )<count_missing_values>
all_logits = torch.cat(predictions, dim=0)
 probs = F.softmax(all_logits, dim=1 ).cpu().numpy()
probs
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )<set_options>
threshold = 0.5
preds = np.where(probs[:, 1] > threshold, 1, 0)
 preds
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
print("Number of tweets labeled as true disaster tweet: ", preds.sum() )
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-transformers-pytorch/roberta-base/vocab.json',
merges_file='.. /input/roberta-transformers-pytorch/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-transformers-pytorch/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.bn1 = nn.BatchNorm1d(num_features=96)
 self.bn2 = nn.BatchNorm1d(num_features=96)
 self.c1=nn.Conv1d(768,768,2)
 self.c11=nn.Conv1d(768,256,2)
 self.c111=nn.Conv1d(256,64,2)
 self.c2=nn.Conv1d(768,768,2)
 self.c22=nn.Conv1d(768,256,2)
 self.c222=nn.Conv1d(256,64,2)
 self.Leaky= nn.ReLU(0.3)
 self.i0=nn.Linear(64,1)
 self.i1=nn.Linear(64,1)
 nn.init.normal_(self.i0.bias, 0)
 nn.init.normal_(self.i0.weight, std=0.02)
 nn.init.normal_(self.i1.bias, 0)
 nn.init.normal_(self.i1.weight, std=0.02)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 out = self.dropout(x)
 out = nn.functional.pad(out.transpose(1,2),(1, 0))
out1 = self.c1(out ).transpose(1,2)
 out1=self.Leaky(self.bn1(out1))
out1 = self.c11(nn.functional.pad(out1.transpose(1,2),(1, 0)) ).transpose(1,2)
 out1=self.Leaky(self.bn2(out1))
out1 = self.c111(nn.functional.pad(out1.transpose(1,2),(1, 0)) ).transpose(1,2)
 out1=self.Leaky(self.bn2(out1))
out2 = self.c2(out ).transpose(1,2)
 out2=self.Leaky(self.bn1(out2))
out2 = self.c22(nn.functional.pad(out2.transpose(1,2),(1, 0)) ).transpose(1,2)
 out2=self.Leaky(self.bn2(out2))
out2 = self.c222(nn.functional.pad(out2.transpose(1,2),(1, 0)) ).transpose(1,2)
 out2=self.Leaky(self.bn2(out2))
start_logits = self.i0(self.dropout(out1)).squeeze(-1)
 end_logits = self.i1(self.dropout(out2)).squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
Y_test = preds
<statistical_test><EOS> 
df_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv', index_col=0 ).fillna('')
 df_submission['target'] = Y_test
df_submission.to_csv('submission.csv')
 !head submission.csv