Kaggle competition: Natural Language Processing with Disaster Tweets
<define_variables>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if len(original_tweet.split())< 2:
filtered_output = original_tweet
if sentiment_val != "neutral" and verbose == True:
if filtered_output.strip().lower() != target_string.strip().lower() :
print("********************************")
 print(f"Output= {filtered_output.strip() }")
 print(f"Target= {target_string.strip() }")
 print(f"Tweet= {original_tweet.strip() }")
 print("********************************")
 jac = 0
return jac, filtered_output<load_from_csv>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv',encoding='UTF-8')
 print('Number of training sentences: {:,}
'.format(train_df.shape[0]))
train_df.sample(10 )
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_from_csv>
with open('.. /input/bert-prediction/predictions.pkl','rb')as f:
predictions = pickle.load(f)
<load_pretrained>
train_df['infold_pred'] = predictions['infold_pred']
train_df['outfold_pred'] = predictions['outfold_pred']
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
<define_variables>
remove_rows_based_on_cv = True
model_name = 'albert'
model_shortcut_name = 'albert-base-v1'
do_lower_case = True
learning_rate = 5e-6
epochs = 6
batch_size = 16
run_test_to_get_best_epoch = False
epochs_final = 2
score_to_take_best = 'val_f1_score'
score_to_take_best_order = False
MODEL_BASE_PATH_OLD = '.. /input/roberta-base-uncased4'
MODEL_BASE_PATH = '.. /input/roberta-pp2'<choose_model_class>
if remove_rows_based_on_cv:
train_df = train_df[(( train_df['target']==0)&(train_df['outfold_pred']<0.3)) |(( train_df['target']==1)&(train_df['outfold_pred']>0.7)) ]
ENSEMBLES = [
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_0.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_1.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_2.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_3.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_4.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_5.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_6.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_7.bin", 'weight': 1},
]<load_pretrained>
train_df.index = range(train_df.shape[0] )
models = []
weights = []
for val in ENSEMBLES:
model = val['model']
model.to(device)
 model.load_state_dict(torch.load(val['state_dict']))
model.eval()
models.append(model)
 weights.append(val['weight'] )<find_best_params>
sentences = train_df.text.values
labels = train_df.target.values
def get_best_start_end_idxs(_start_logits, _end_logits):
best_logit = -1000
best_idxs = None
for start_idx, start_logit in enumerate(_start_logits):
for end_idx, end_logit in enumerate(_end_logits[start_idx:]):
logit_sum =(start_logit + end_logit ).item()
if logit_sum > best_logit:
best_logit = logit_sum
best_idxs =(start_idx, start_idx+end_idx)
 return best_idxs<load_pretrained>
if model_name == 'roberta':
print('Loading RoBERTa tokenizer...')
 tokenizer = RobertaTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case)
 elif model_name == 'bert':
print('Loading BERT tokenizer...')
 tokenizer = BertTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case)
 elif model_name == 'albert':
print('Loading AlBERT tokenizer...')
 tokenizer = AlbertTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case )
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start = []
outputs_end = []
for index, model in enumerate(models):
output_start, output_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start.append(output_start * weights[index])
 outputs_end.append(output_end * weights[index])
 outputs_start = sum(outputs_start)/ len(outputs_start)
 outputs_end = sum(outputs_end)/ len(outputs_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
idx_start, idx_end = get_best_start_end_idxs(outputs_start[px, :], outputs_end[px, :])
 _, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=idx_start,
idx_end=idx_end,
offsets=offsets[px] 
)
 final_output.append(output_sentence )<feature_engineering>
print(' Original: ', sentences[0])
 print('Tokenized: ', tokenizer.tokenize(sentences[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])) )
df_test['selected_text'] = final_output
df_test['selected_text'] = df_test.apply(lambda x: postprocess(x.text, x.selected_text), axis=1 )<string_transform>
max_len = 0
for sent in sentences:
input_ids = tokenizer.encode(sent, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max sentence length: ', max_len )
def place_in_back(x):
splitted = x.text.split(x.selected_text)[0]
sub = len(splitted)- len(" ".join(splitted.split()))
select=x.selected_text
ind = x.text.find(x.selected_text.strip())-1
if(( ind >0)&(select.startswith('.')!=1)&(sub>0)) :
if(( x.text[ind] in string.punctuation)&(x.sentiment!='neutral')&(sub==1)) :
select = x.text[ind]+select
print(select,sub)
 elif(( x.text[ind-1] in string.punctuation)&(x.sentiment!='neutral')&(sub==2)) :
select = x.text[ind-1]+""+select
print(select,sub)
 return select 
<feature_engineering>
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 100,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[1])
 print('Token IDs:', input_ids[1] )
df_test['selected_text']=df_test.apply(lambda x : place_in_back(x),axis=1 )<feature_engineering>
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.9 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 label_temp_list = []
for a,b,c in train_dataset:
label_temp_list.append(c)
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} training samples with real disater tweets'.format(sum(label_temp_list)))
 label_temp_list = []
for a,b,c in val_dataset:
label_temp_list.append(c)
 print('{:>5,} validation samples'.format(val_size))
print('{:>5,} validation samples with real disater tweets'.format(sum(label_temp_list)) )
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = df_test['selected_text']
sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('!!!', '!')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('!!!!', '!!')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('.... ', '.. ')if len(x.split())==1 else x)
 sample.to_csv("submission.csv", index=False)
<import_modules>
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 full_train_dataloader = DataLoader(
 dataset,
sampler = SequentialSampler(dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)
import os
import torch
import pandas as pd
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn.model_selection import KFold
from sklearn import metrics
from transformers import RobertaModel, RobertaConfig
import tokenizers
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from tqdm.autonotebook import tqdm
import utils<define_variables>
def start_model(model_name,model_shortcut_name):
if model_name == 'bert':
model = BertForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 elif model_name == 'roberta':
model = RobertaForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 elif model_name == 'albert':
model = AlbertForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 try:
model.cuda()
except:
pass
return model
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 16
EPOCHS = 3
MODEL_PATH = ".. /input/roberta-base/pytorch_model.bin"
TRAINING_FILE = ".. /input/tweet-sentiment-extraction/train.csv"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.drop_out = nn.Dropout(0.2)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
def train_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):
model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg<load_from_csv>
model = start_model(model_name,model_shortcut_name )
def run(fold):
dfx = pd.read_csv(config.TRAINING_FILE)
 kf = KFold(n_splits = 10, shuffle = True, random_state = 42)
 kf_r = next(kf.split(dfx), None)
 df_train = dfx.iloc[kf_r[0]].reset_index(drop=True)
 df_valid = dfx.iloc[kf_r[1]].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model = TweetModel()
model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = utils.EarlyStopping(patience=2, mode="max")
 for epoch in range(3):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"modelo_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<train_model>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
for fold in range(3):
print(f"Fold={fold}")
 run(fold )<find_best_params>
def get_optimizer_and_scheduler(learning_rate,train_dataloader,epochs):
optimizer = AdamW(model.parameters() ,
lr = learning_rate,
eps = 1e-8 
)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps)
 return optimizer,scheduler
device = torch.device("cuda")
 modelo1 = TweetModel()
modelo1.to(device)
 modelo1.load_state_dict(torch.load("modelo_0.bin"))
modelo1.eval()
modelo2 = TweetModel()
modelo2.to(device)
 modelo2.load_state_dict(torch.load("modelo_1.bin"))
modelo2.eval()
modelo3 = TweetModel()
modelo3.to(device)
 modelo3.load_state_dict(torch.load("modelo_2.bin"))
modelo3.eval() 
<load_from_csv>
optimizer,scheduler = get_optimizer_and_scheduler(learning_rate,train_dataloader,epochs )
final_output = []
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<create_dataframe>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = modelo1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = modelo2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = modelo3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(outputs_start1 + outputs_start2 + outputs_start3)/ 3
outputs_end =(outputs_end1 + outputs_end2 + outputs_end3)/ 3
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence)
<save_to_csv>
def flat_f1_score(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return f1_score(labels_flat,pred_flat )
submission = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 submission.loc[:, 'selected_text'] = final_output
submission.to_csv("submission.csv", index=False )<import_modules>
if run_test_to_get_best_epoch:
seed_val = 66
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
logits_list = []
labels_list = []
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 :
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 logits_list.append(logits.detach().cpu().numpy())
 labels_list.append(b_labels.to('cpu' ).numpy())
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_accuracy = flat_accuracy(np.concatenate(logits_list),np.concatenate(labels_list),)
 training_f1_score = flat_f1_score(np.concatenate(logits_list),np.concatenate(labels_list),)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
logits_list = []
labels_list = []
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 logits_list.append(logits.detach().cpu().numpy())
 labels_list.append(b_labels.to('cpu' ).numpy())
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
testing_accuracy = flat_accuracy(np.concatenate(logits_list),np.concatenate(labels_list),)
 testing_f1_score = flat_f1_score(np.concatenate(logits_list),np.concatenate(labels_list),)
 avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time,
'train_accuracy':training_accuracy,
'train_f1_score':training_f1_score,
'val_accuracy':testing_accuracy,
'val_f1_score':testing_f1_score,
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )
import numpy as np
import pandas as pd
import os
import tokenizers
import string
import torch
import transformers
import torch.nn as nn
from torch.nn import functional as F
from tqdm.autonotebook import tqdm
import re
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup 
<define_variables>
if run_test_to_get_best_epoch:
df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 print(df_stats.to_csv() )
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 32
EPOCHS = 3
ELECTRA_PATH = "/kaggle/input/electrabase/electra/base/model"
MODEL_PATH = "pytorch_model.bin"
TRAINING_FILE = ".. /input/tweet-train-folds/train_folds.csv"
TOKENIZER = tokenizers.BertWordPieceTokenizer(
 "/kaggle/input/berthub/assets/vocab.txt",
lowercase=True
)<define_variables>
if run_test_to_get_best_epoch:
pd.set_option('precision', 3)
 df_stats
PATH="/kaggle/output"<string_transform>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<init_hyperparams>
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
class EarlyStopping:
def __init__(self, patience=20, mode="max", delta=0.001):
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score + self.delta:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved({} --> {} ).Saving model!'.format(self.val_score, epoch_score))
torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score 
<feature_engineering>
if run_test_to_get_best_epoch:
best_epoch = df_stats.sort_values(score_to_take_best,ascending=score_to_take_best_order ).head(1 ).index[0]
if epochs_final:
best_epoch = epochs_final
print(best_epoch )
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
len_st = len(selected_text)
 idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[0]):
if tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids[1:-1]
tweet_offsets = tok_tweet.offsets[1:-1]
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 3893,
'negative': 4997,
'neutral': 8699
}
input_ids = [101] + [sentiment_id[sentiment]] + [102] + input_ids_orig + [102]
token_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 3 + tweet_offsets + [(0, 0)]
targets_start += 3
targets_end += 3
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}<define_variables>
model = start_model(model_name,model_shortcut_name)
 optimizer,scheduler = get_optimizer_and_scheduler(learning_rate,full_train_dataloader,best_epoch )
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<normalization>
seed_val = 66
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, best_epoch):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, best_epoch))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(full_train_dataloader):
if step % 40 == 0 :
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Training Time': training_time,
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.bert = transformers.ElectraModel.from_pretrained(config.ELECTRA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.2)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.002)
 def forward(self, ids, mask, token_type_ids):
_,out = self.bert(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<train_on_grid>
df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print('Number of test sentences: {:,}
'.format(df.shape[0]))
sentences = df.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 75,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
def train_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
losses = AverageMeter()
jaccards = AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
<define_variables>
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print(' DONE.' )
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):
model.eval()
losses = AverageMeter()
jaccards = AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg<load_from_csv>
flat_predictions = np.concatenate(predictions, axis=0)
 flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
df['target'] = flat_predictions
def run(fold):
dfx = pd.read_csv(config.TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model_config = transformers.ElectraConfig.from_pretrained(config.ELECTRA_PATH)
 model_config.output_hidden_states = True
model = TweetModel(conf=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = EarlyStopping(patience=2, mode="max")
 print(f"Training is Starting for fold={fold}")
 for epoch in range(config.EPOCHS):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"example_run_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<load_from_csv>
submission['target'] = flat_predictions
submission.to_csv('torch_prediction.csv', index=False )
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<normalization>
device = torch.device("cuda")
 model_config = transformers.ElectraConfig.from_pretrained(config.ELECTRA_PATH)
 model_config.output_hidden_states = True<load_pretrained>
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/tweetmodel1/example_run_5(1 ).bin"))
model1.eval() 
<create_dataframe>
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<string_transform>
<save_to_csv><EOS>