Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<load_from_zip>
import numpy as np
import pandas as pd
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np
if not is_inference_flag:
!git clone https://github.com/AnandAwasthi/Closed-domain-Question-Answering-fine-tune-Albert 
<import_modules>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
train_tasks = sorted(os.listdir(training_path))
eval_tasks = sorted(os.listdir(evaluation_path))
import pandas as pd
import numpy as np
import json
from sklearn.model_selection import train_test_split<load_from_csv>
train_file = str(training_path / train_tasks[15])
 with open(train_file, 'r')as f:
task = json.load(f )
train_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 sub_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
<string_transform>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def jaccard(str1, str2):
a = str1.lower().split()
b = str2.lower().split()
c = set(a ).intersection(set(b))
return float(len(c)) /(len(a)+ len(b)- len(c))<correct_missing_values>
def find_count_tow_list(a,b):
count = 0
for i,j in enumerate(a):
if j == b[i]:
count+=1
return count
train_df.dropna(inplace=True )<split>
replace_color = {}
for k in range(len(task['train'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
data = [[replace_color[i] for i in j] for j in task['test'][0]['input']]
X_train, X_test = train_test_split(train_df,test_size=0.10, random_state=42 )<prepare_x_and_y>
train_correct = len(train_tasks)
 for i in range(len(train_tasks)) :
train_file = str(training_path / train_tasks[i])
 with open(train_file, 'r')as f:
task = json.load(f)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][0]['input']]
pred_1 = flattener(data)
 if pred_1 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 train_correct-=1
for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 if pred_2 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 train_correct-=1
train = np.array(X_train)
 val = np.array(X_test)
 test = np.array(test_df)
 use_cuda = True<categorify>
print("Training set error rate ==> {}, detect tasks {} / {}".format(train_correct/len(train_tasks),len(train_tasks)- train_correct,len(train_tasks)) )
%%time

 def convert_test_qa_json(test):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return output
qa_test = convert_test_qa_json(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
<define_variables>
eval_correct = len(eval_tasks)
 for i in range(len(eval_tasks)) :
eval_file = str(evaluation_path / eval_tasks[i])
 with open(eval_file, 'r')as f:
task = json.load(f)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][0]['input']]
pred_1 = flattener(data)
 if pred_1 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 eval_correct-=1
for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 if pred_2 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 eval_correct-=1
if not is_inference_flag:
!export SQUAD_DIR=data \
&& python Closed-domain-Question-Answering-fine-tune-Albert/bsie/transformers/examples/run_squad.py \
--model_type albert \
--model_name_or_path albert-base-v2 \
--do_train \
--do_eval \
--do_lower_case \
--train_file $SQUAD_DIR/train.json \
--predict_file $SQUAD_DIR/val.json \
--per_gpu_train_batch_size 12 \
--learning_rate 5e-5 \
--num_train_epochs 1.0 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir $SQUAD_DIR/models/albert \
--save_steps 100000 \
--threads 4 \
--version_2_with_negative \
--overwrite_output_dir<define_variables>
print("Evaluate set error rate ==> {}, detect tasks {} / {}".format(eval_correct/len(eval_tasks),len(eval_tasks)- eval_correct,len(eval_tasks)) )
AutoConfig,
AutoModelForQuestionAnswering,
AutoTokenizer,
squad_convert_examples_to_features 
)
 if not is_inference_flag:
model_name_or_path = "data/models/"
else:
model_name_or_path = '/kaggle/input/albert-xl-300/'
output_dir = ""
n_best_size = 1
max_answer_length = 254
do_lower_case = True
null_score_diff_threshold = 0.0
def to_list(tensor):
return tensor.detach().cpu().tolist()
config_class, model_class, tokenizer_class =(
 AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer)
 config = config_class.from_pretrained(model_name_or_path)
 tokenizer = tokenizer_class.from_pretrained(
 model_name_or_path, do_lower_case=True)
 model = model_class.from_pretrained(model_name_or_path, config=config)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device)
 processor = SquadV2Processor()
def run_prediction(question_texts, context_text):

 if question_texts[0] != 'neutral':
examples = []
for i, question_text in enumerate(question_texts):
example = SquadExample(
 qas_id=str(i),
question_text=question_text,
context_text=context_text,
answer_text=None,
start_position_character=None,
title="Predict",
is_impossible=False,
answers=None, 
)
 examples.append(example)
 features, dataset = squad_convert_examples_to_features(
 examples=examples,
tokenizer=tokenizer,
max_seq_length=300,
doc_stride=128,
max_query_length=64,
is_training=False,
return_dataset="pt",
threads=1, 
)
 eval_sampler = SequentialSampler(dataset)
 eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=10)
 all_results = []
for batch in eval_dataloader:
model.eval()
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"token_type_ids": batch[2],
}
example_indices = batch[3]
outputs = model(**inputs)
 for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item() ]
unique_id = int(eval_feature.unique_id)
 output = [to_list(output[i])for output in outputs]
start_logits, end_logits = output
result = SquadResult(unique_id, start_logits, end_logits)
 all_results.append(result)
 output_prediction_file = "predictions.json"
output_nbest_file = "nbest_predictions.json"
output_null_log_odds_file = "null_predictions.json"
predictions = compute_predictions_logits(
 examples,
features,
all_results,
n_best_size,
max_answer_length,
do_lower_case,
output_prediction_file,
output_nbest_file,
output_null_log_odds_file,
False,
True,
null_score_diff_threshold,
tokenizer, 
)
 else:
predictions = OrderedDict([(0, context_text)])
 return predictions<compute_test_metric>
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )
<save_to_csv><EOS> 
count = 0
for output_id in submission.index:
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = str(test_path / str(task_id + '.json'))
with open(f, 'r')as read_file:
task = json.load(read_file)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][pair_id]['input']]
pred_1 = flattener(data)
 for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 data = [[0 for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv' )