Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<choose_model_class>
import json
import re
import os
import numpy as np
import pandas as pd
class TweetRoberta(pl.LightningModule):
def __init__(self, roberta_path, fold_i=None, train_len=None):
super().__init__()
self.fold_i = fold_i
self.train_len = train_len
self.model = TweetRobertaModel(roberta_path)
 self.am_tloss = AverageMeter()
self.am_vloss = AverageMeter()
self.am_jscore = AverageMeter()
def _compute_loss(self, start_logits, end_logits, batch):
ignore_ix = start_logits.size(1)
 start_logits.clamp_(0., ignore_ix)
 end_logits.clamp_(0., ignore_ix)
 loss_fn = nn.CrossEntropyLoss(ignore_index=ignore_ix)
 start_loss = loss_fn(start_logits, batch['start_token'])
 end_loss = loss_fn(end_logits, batch['end_token'])
 return start_loss + end_loss
def _jaccard_score(self, text1, text2):
a = set(text1.lower().split())
 b = set(text2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def _convert_tokens_to_text(self, start_token, end_token, text, sentiment, offsets):
if end_token < start_token: end_token = start_token
filtered_text = ''
for i in range(start_token, end_token+1):
filtered_text += text[offsets[i, 0]:offsets[i, 1]]
if(( i+1)< len(offsets)) and(offsets[i, 1] < offsets[i+1, 0]): filtered_text += ' '
if(sentiment == 'neutral')or(len(text.split())< 2): filtered_text = text
return filtered_text
def batch_convert_logits_to_text(self, start_logits, end_logits, batch, return_score=True):
filtered_texts = []
jac_scores = []
start_token_preds = torch.argmax(start_logits, dim=1)
 end_tokens_preds = torch.argmax(end_logits, dim=1)
 for i, text in enumerate(batch['orig_text']):
sentiment = batch['sentiment'][i]
start_token_pred = start_token_preds[i]
end_token_pred = end_tokens_preds[i]
filtered_text = self._convert_tokens_to_text(
 start_token_pred, end_token_pred, text,
sentiment, batch['offsets'][i] 
)
 if return_score:
selected_text = batch['orig_selected_text'][i]
jac = self._jaccard_score(selected_text.strip() , filtered_text.strip())
 jac_scores.append(jac)
 else:
filtered_texts.append(filtered_text)
 if return_score: return torch.tensor(jac_scores ).mean()
else: return filtered_texts
def forward(self, batch):
return self.model(batch['input_ids'], batch['attn_mask'], batch['token_type_ids'])
 def training_step(self, batch, batch_nb):
start_logits, end_logits = self.forward(batch)
 train_loss = self._compute_loss(start_logits, end_logits, batch)
 self.am_tloss.update(train_loss.item() , len(batch['input_ids']))
return {'loss': train_loss, 'log': {f'train_loss_{self.fold_i}': train_loss}}
def validation_step(self, batch, batch_nb):
start_logits, end_logits = self.forward(batch)
 start_logits, end_logits = start_logits.detach() , end_logits.detach()
valid_loss = self._compute_loss(start_logits, end_logits, batch)
 jac_score = self.batch_convert_logits_to_text(start_logits, end_logits, batch)
 self.am_vloss.update(valid_loss.item() , len(batch['input_ids']))
self.am_jscore.update(jac_score, len(batch['input_ids']))
def validation_epoch_end(self, _):
tqdm_dict = {
f'avg_train_loss_{self.fold_i}': self.am_tloss.avg,
f'avg_valid_loss_{self.fold_i}': self.am_vloss.avg,
f'avg_jaccard_{self.fold_i}': self.am_jscore.avg
}
self.am_tloss.reset()
self.am_vloss.reset()
self.am_jscore.reset()
return {'progress_bar': tqdm_dict, 'log': tqdm_dict}
def predict(self, batch, device, mode='test'):
self.eval()
with torch.no_grad() :
batch = move_to_device(batch, device)
 start_logits, end_logits = self.forward(batch)
 return start_logits.detach() , end_logits.detach()
def configure_optimizers(self):
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.weight']
optimizer_parameters = [
{
'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay': 1e-3
},
{
'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay': 0.
 }
]
optimizer = optim.AdamW(optimizer_parameters, lr=GCONF.lr)
 train_steps =(self.train_len * GCONF.epochs)//GCONF.accum_steps
scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=int(GCONF.warmup_steps*train_steps),
num_training_steps=train_steps 
)
 scheduler_dict = {
'scheduler': scheduler,
'interval': 'step'
}
return [optimizer], [scheduler_dict]<load_pretrained>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
wandb_logger = pl.loggers.WandbLogger(
 name='roberta_uncased',
save_dir='/kaggle/working/',
project='kaggle_tweet_extraction',
offline=GCONF.offline 
)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 tokenizer = RobertaTokenizerFast.from_pretrained(str(GCONF.roberta_path), do_lower_case=True, add_prefix_space=True )<split>
testDirectory = "/kaggle/input/abstraction-and-reasoning-challenge/test/"
skf = model_selection.StratifiedKFold(n_splits=GCONF.n_splits, shuffle=True, random_state=GCONF.seed)
 for fold_i,(train_ix, valid_ix)in enumerate(skf.split(train_df, train_df['sentiment'].values)) :
train_ds = TweetRobertaDataset(train_df.iloc[train_ix], tokenizer, GCONF.max_length, is_testing=False)
 train_dl = DataLoader(train_ds, batch_size=GCONF.batch_size, shuffle=True)
 valid_ds = TweetRobertaDataset(train_df.iloc[valid_ix], tokenizer, GCONF.max_length, is_testing=False)
 valid_dl = DataLoader(valid_ds, batch_size=GCONF.batch_size*4, shuffle=False)
 mc_cb, es_cb = get_callbacks(fold_i)
 model = TweetRoberta(GCONF.roberta_path, fold_i, len(train_dl))
trainer = pl.Trainer(
 checkpoint_callback=mc_cb,
early_stop_callback=es_cb,
logger=wandb_logger,
gpus='0',
max_epochs=GCONF.epochs,
accumulate_grad_batches=GCONF.accum_steps, 
)
 trainer.fit(model, train_dl, valid_dl)
 torch.cuda.empty_cache()
best_model_fn = get_best_model_fn(mc_cb, fold_i)
 trainer.restore(best_model_fn, on_gpu=False)
 torch.save(model.state_dict() , GCONF.saved_model_path/Path(best_model_fn ).name)
 model.to('cpu')
 del train_ds, valid_ds, mc_cb, es_cb, model, trainer
gc.collect()
torch.cuda.empty_cache()
shutil.rmtree('/kaggle/working/models' )<load_from_csv>
def readTaskFile(filename):
f = open(filename, "r")
 data = json.loads(f.read())
 data["id"] = re.sub(" (.*/)|(\.json)", "", filename)
 f.close()
return data
test_df = pd.read_csv(GCONF.path/'test.csv')
 print(test_df.shape)
 test_df.head()<feature_engineering>
filename = testDirectory+"3b4c2228.json"
readTaskFile(filename )
test_df['text'] = test_df['text'].apply(remove_space )<create_dataframe>
def getNoopAnswer(filename):
data = readTaskFile(filename)
 testSection = data["test"]
ident = data["id"]
numTests = len(testSection)
 answer = {}
for i in range(numTests):
answer[ident+"_"+str(i)] = flattener(testSection[i]["input"])
 return answer
test_ds = TweetRobertaDataset(test_df, tokenizer, GCONF.max_length, is_testing=True)
 test_dl = DataLoader(test_ds, batch_size=GCONF.batch_size*4, shuffle=False )<load_pretrained>
filename = testDirectory+"3b4c2228.json"
getNoopAnswer(filename )
models = []
for mf_path in tqdm(GCONF.saved_model_path.iterdir()):
model = TweetRoberta(GCONF.roberta_path)
 model.load_state_dict(torch.load(mf_path, map_location=lambda storage, loc: storage))
model.to(device)
 models.append(model )<predict_on_test>
def getAnswers(directory, f):
answers = {}
for _, _, filenames in os.walk(directory):
for filename in filenames:
answers.update(f(directory+filename))
return answers
test_results = []
for batch in tqdm(test_dl):
start_logits, end_logits = 0, 0
for model in models:
tmp_start_logits, tmp_end_logits = model.predict(batch, device)
 start_logits += tmp_start_logits
end_logits += tmp_end_logits
start_logits /= len(models)
 end_logits /= len(models)
 pred_texts = model.batch_convert_logits_to_text(start_logits, end_logits, batch, return_score=False)
 test_results.extend(pred_texts )<save_to_csv>
answers = getAnswers(testDirectory, getNoopAnswer)
 f = open("submission.csv", "w")
 f.write("output_id,output
")
 keys = answers.keys()
for key in keys:
value = answers[key]
f.write(key + "," + value + "
")
 f.close()
sub_df = pd.read_csv(GCONF.path/'sample_submission.csv')
 sub_df.loc[:, 'selected_text'] = test_results
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
f2 = open("submission.csv", "r")
 print(f2.read())
 f2.close()