Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<predict_on_test>
warnings.filterwarnings('ignore' )
pred_test = model.predict(test_x )<save_to_csv>
cuda_yes = torch.cuda.is_available()
print('Cuda is available?', cuda_yes)
 device = torch.device("cuda:0" if cuda_yes else "cpu")
 print('Device:', device)
submission.to_csv('submission.csv', index=False )<save_to_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )
submission.to_csv('submission.csv', index=False )<import_modules>
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
data['sentiment'] = row.sentiment
if self.labeled:
start_idx, end_idx,labels = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
data['labels']=labels
data['selected_text']=row.selected_text
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 if row.sentiment=="positive":
sent=1
elif row.sentiment=="negative":
sent=2
else:
sent=3
len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 is not None and idx1 is not None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
labels = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 labels.append(sent)
 elif j==1:
labels.append(sent)
 else:
labels.append(0)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
labels = torch.tensor(labels)
 return start_idx, end_idx, labels
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from numpy import savetxt<load_from_csv>
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True,num_labels=NUM_LABELS)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.num_labels = config.num_labels
self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask, labels=None):
sequence_output, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3],hs[-4]])
 x = torch.mean(x, 0)
 x_drop = self.dropout(x)
 x = self.fc(x_drop)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 logits = self.classifier(x_drop)
 outputs=(logits,start_logits, end_logits)
 if labels is not None:
loss_fct = nn.CrossEntropyLoss()
if attention_mask is not None:
active_loss = attention_mask.view(-1)== 1
active_logits = logits.view(-1, self.num_labels)
 active_labels = torch.where(
 active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index ).type_as(labels)
)
 loss = loss_fct(active_logits, active_labels)
 else:
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs
base_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 base_train = base_train[['PassengerId', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked', 'Survived']]
del base_train['Name']
del base_train['Ticket']
del base_train['Cabin']
base_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 del base_test['Name']
del base_test['Ticket']
del base_test['Cabin']<categorify>
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss
imputer = SimpleImputer(missing_values = np.nan, strategy = 'mean')
 imputer = imputer.fit(base_train.iloc[:, 3:4])
 base_train.iloc[:, 3:4] = imputer.transform(base_train.iloc[:, 3:4])
 base_test.iloc[:, 3:4] = imputer.transform(base_test.iloc[:, 3:4] )<categorify>
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text,selected_text,start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 jaccard_approx = jaccard(true, pred)
 jaccard_true = jaccard(selected_text,pred)
 return jaccard_approx,jaccard_true
labelencoder = LabelEncoder()
base_train['Embarked'] = labelencoder.fit_transform(base_train['Embarked'].astype(str))
base_test['Embarked'] = labelencoder.fit_transform(base_test['Embarked'].astype(str))
base_train.iloc[:, 2] = labelencoder.fit_transform(base_train.iloc[:, 2])
 base_test.iloc[:, 2] = labelencoder.fit_transform(base_test.iloc[:, 2] )<normalization>
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.to(device)
 for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_approx_jaccard = 0.0
epoch_true_jaccard = 0.0
epoch_start_end_loss=0.0
epoch_ner_loss=0.0
tk0 = tqdm(dataloaders_dict[phase], total=len(dataloaders_dict[phase]))
for data in tk0:
ids = data['ids'].to(device)
 masks = data['masks'].to(device)
 tweet = data['tweet']
selected_text=data['selected_text']
sentiment = data['sentiment']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].to(device)
 end_idx = data['end_idx'].to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
ner_loss,logits,start_logits, end_logits = model(ids, masks,labels)
 start_end_loss = criterion(start_logits, end_logits, start_idx, end_idx)
 loss=ner_loss+start_end_loss
if phase == 'train':
loss.backward()
optimizer.step()
scheduler.step()
epoch_loss += loss.item() * len(ids)
 epoch_start_end_loss += start_end_loss.item() * len(ids)
 epoch_ner_loss += ner_loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_approx_score,jaccard_true_score = compute_jaccard_score(
 tweet[i],
selected_text[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_approx_jaccard += jaccard_approx_score
epoch_true_jaccard += jaccard_true_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_ner_loss = epoch_ner_loss / len(dataloaders_dict[phase].dataset)
 epoch_start_end_loss = epoch_start_end_loss / len(dataloaders_dict[phase].dataset)
 epoch_approx_jaccard = epoch_approx_jaccard / len(dataloaders_dict[phase].dataset)
 epoch_true_jaccard = epoch_true_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | NER_Loss: {:.4f} | Start_End_Loss: {:.4f} | Approx_Jaccard: {:.4f} | True_Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss,epoch_ner_loss,epoch_start_end_loss,epoch_approx_jaccard,epoch_true_jaccard))
torch.save(model.state_dict() , filename )
scaler = StandardScaler()
base_train.iloc[:, 1:8] = scaler.fit_transform(base_train.iloc[:, 1:8])
 base_test.iloc[:, 1:8] = scaler.fit_transform(base_test.iloc[:, 1:8] )<prepare_x_and_y>
num_epochs = 3
batch_size = 32
gradient_accumulation_steps = 1
warmup_proportion=0.1
NUM_LABELS=4
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )
base_test[base_test==np.inf]=np.nan
base_test.fillna(base_test.mean() , inplace=True )<split>
predictors_train = base_train.iloc[:, 0:8].values
class_train = base_train.iloc[:, 8].values<predict_on_test>
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
model_dir = ".. /input/tweet-sentiment-roberta-pytorch/"
for fold in range(skf.n_splits):
model = TweetModel()
model.to(device)
 model.load_state_dict(torch.load(model_dir+f'roberta_fold{fold+1}.pth',map_location=torch.device(device)))
 model.eval()
models.append(model)
 for data in tqdm(test_loader):
ids = data['ids'].to(device)
 masks = data['masks'].to(device)
 tweet = data['tweet']
sentiment = data['sentiment']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[2], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if(start_pred > end_pred):
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )
<save_to_csv><EOS> 
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()