Kaggle competition: Abstraction and Reasoning Challenge
print('TF version',tf.__version__ )<load_from_csv>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(len(training_tasks), len(evaluation_tasks), len(test_tasks))
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
def get_data(task_filename):
with open(task_filename, 'r')as f:
task = json.load(f)
 return task
num2color = ["black", "blue", "red", "green", "yellow", "gray", "magenta", "orange", "sky", "brown"]
color2num = {c: n for n, c in enumerate(num2color)}
def jaccard_improve(str1, str2):
str1=str1.lower()
str2=str2.lower()
index=str1.find(str2)
 text1=str1[:index]
text2=str1[index:].replace(str2,'')
 words1=text1.split()
words2=text2.split()
if len(words1)>len(words2):
words1=words1[-3:]
mod_text=" ".join(words1)+" "+ str2
else:
words2=words2[0:2]
mod_text=str2+" "+" ".join(words2)
 return mod_text<string_transform>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<feature_engineering>
sample_sub = pd.read_csv(data_path/'sample_submission.csv')
 sample_sub = sample_sub.set_index('output_id')
 sample_sub.head()
train_df['selected_text_mod']=train_df['selected_text']
train_df['mod']=0<define_variables>
def check(task, learn_func, pred_func):
n = len(task["train"])+ len(task["test"])
 fig, axs = plt.subplots(3, n, figsize=(4*n,12), dpi=50)
 plt.subplots_adjust(wspace=0.3, hspace=0.3)
 fig_num = 0
param_ls=[]
for i, t in enumerate(task["train"]):
t_in, t_out = np.array(t["input"]), np.array(t["output"])
 t_pred, param = learn_func(t_in,t_out)
 if param==False:
print('Not suitable')
 plt.close('all')
 return
param_ls.append(param)
 axs[0][fig_num].imshow(t_in, cmap=cmap, norm=norm)
 axs[0][fig_num].set_title(f'Train-{i} in')
 axs[0][fig_num].set_yticks(list(range(t_in.shape[0])))
 axs[0][fig_num].set_xticks(list(range(t_in.shape[1])))
 axs[1][fig_num].imshow(t_out, cmap=cmap, norm=norm)
 axs[1][fig_num].set_title(f'Train-{i} out')
 axs[1][fig_num].set_yticks(list(range(t_out.shape[0])))
 axs[1][fig_num].set_xticks(list(range(t_out.shape[1])))
 axs[2][fig_num].imshow(t_pred, cmap=cmap, norm=norm)
 axs[2][fig_num].set_title(f'Train-{i} pred')
 axs[2][fig_num].set_yticks(list(range(t_pred.shape[0])))
 axs[2][fig_num].set_xticks(list(range(t_pred.shape[1])))
 fig_num += 1
if len(set(param_ls)) ==1:
param=param_ls[0]
for i, t in enumerate(task["test"]):
t_in = np.array(t["input"])
 t_pred = pred_func(t_in,param)
 axs[0][fig_num].imshow(t_in, cmap=cmap, norm=norm)
 axs[0][fig_num].set_title(f'Test-{i} in')
 axs[0][fig_num].set_yticks(list(range(t_in.shape[0])))
 axs[0][fig_num].set_xticks(list(range(t_in.shape[1])))
 axs[1][fig_num].imshow(t_pred, cmap=cmap, norm=norm)
 axs[1][fig_num].set_title(f'Test-{i} pred')
 axs[1][fig_num].set_yticks(list(range(t_pred.shape[0])))
 axs[1][fig_num].set_xticks(list(range(t_pred.shape[1])))
 fig_num += 1
print(t_pred)
 if mode=='test':
print('mode test')
 sub_ls=flattener(t_pred.tolist())
 sample_sub.loc[f'{test_tasks[iteration][:-5]}_{i}','output'] = sample_sub.loc[f'{test_tasks[iteration][:-5]}_{i}','output'] +' '+sub_ls
plt.show()
else:
print('NOT THIS TYPE')
 return
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
def flip_rot(matrix_pred,matrix_out):
flip_0=np.flip(matrix_pred, 0)
 matrix_out=matrix_out.tolist()
matrix_out=flattener(matrix_out)
 if matrix_out==flattener(matrix_pred.tolist()):
print('!!!match1!!!')
 return 'y'
elif matrix_out==flattener(( np.rot90(matrix_pred)).tolist()):
print('!!!match2!!!')
 return 'np.rot90(y)'
elif matrix_out==flattener(( np.rot90(matrix_pred, 2)).tolist()):
print('!!!match3!!!')
 return 'np.rot90(y, 2)'
elif matrix_out==flattener(( np.rot90(matrix_pred, 3)).tolist()):
print('!!!match4!!!')
 return 'np.rot90(y, 3)'
elif matrix_out==flattener(flip_0.tolist()):
print('!!!match5!!!')
 return 'np.flip(y, 0)'
elif matrix_out==flattener(( np.rot90(flip_0)).tolist()):
print('!!!match6!!!')
 return 'np.rot90(np.flip(y, 0)) '
elif matrix_out==flattener(( np.rot90(flip_0, 2)).tolist()):
print('!!!match7!!!')
 return 'np.rot90(np.flip(y, 0),2)'
elif matrix_out==flattener(( np.rot90(flip_0, 3)).tolist()):
print('!!!match8!!!')
 return 'np.rot90(np.flip(y, 0),3)'
else:
return 'y'
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32' )<define_variables>
def pattern_1_learn(x,t_out):
def split_by_gray_line(arr):
H, W = arr.shape
Y = [-1]
for y in range(H):
if(arr[y, :]==color3 ).all() :
Y.append(y)
 Y.append(H)
 X = [-1]
for x in range(W):
if(arr[:, x]==color3 ).all() :
X.append(x)
 X.append(W)
 res = [[arr[y1+1:y2, x1+1:x2] for x1, x2 in zip(X[:-1], X[1:])] for
y1, y2 in zip(Y[:-1], Y[1:])]
return res
def change_color(arr, d):
res = arr.copy()
for k, v in d.items() :
res[arr==k] = v
return res
for color2 in range(10):
for color3 in range(10):
for color5 in range(10):
for color6 in range(10):
x_split = split_by_gray_line(x)
 if len(x_split)==1 and len(x_split[0])==2:
x1, x2 = x_split[0]
elif len(x_split)==2 and len(x_split[0])==1:
x1, x2 = x_split
x1=x1[0]
x2=x2[0]
else:
continue
if x1.shape!=x2.shape:
continue
rules=['operator.or_(x1,x2)','operator.and_(x1,x2)',
'operator.xor(x1,x2)']
for rule in rules:
zero_one = lambda t: 0 if(t == color5)else 1
vfunc = np.vectorize(zero_one)
 try:
x1=vfunc(x1)
 x2=vfunc(x2)
 except ValueError as e:
print(e)
 continue
try:
y = eval(rule)
 except ValueError as e:
continue
change_color_v2 = lambda t: color6 if(t == 0)else color2
vfunc = np.vectorize(change_color_v2)
 y=vfunc(y)
 f_r=flip_rot(y,t_out)
 y=eval(f_r)
 if flattener(t_out.tolist())==flattener(y.tolist()):
param=(color2,color3,color5,color6,rule,f_r)
 return y, param
return _, False
def pattern_1_pred(x,param):
color2,color3,color5,color6,rule,f_r = param
def split_by_gray_line(arr):
H, W = arr.shape
Y = [-1]
for y in range(H):
if(arr[y, :]==color3 ).all() :
Y.append(y)
 Y.append(H)
 X = [-1]
for x in range(W):
if(arr[:, x]==color3 ).all() :
X.append(x)
 X.append(W)
 res = [[arr[y1+1:y2, x1+1:x2] for x1, x2 in zip(X[:-1], X[1:])] for
y1, y2 in zip(Y[:-1], Y[1:])]
return res
def change_color(arr, d):
res = arr.copy()
for k, v in d.items() :
res[arr==k] = v
return res
x_split = split_by_gray_line(x)
 if len(x_split)==1 and len(x_split[0])==2:
x1, x2 = x_split[0]
elif len(x_split)==2 and len(x_split[0])==1:
x1, x2 = x_split
x1=x1[0]
x2=x2[0]
else:
print('errrrrr')
 zero_one = lambda t: 0 if(t == color5)else 1
vfunc = np.vectorize(zero_one)
 x1=vfunc(x1)
 x2=vfunc(x2)
 try:
y = eval(rule)
 except ValueError as e:
print(e)
 change_color_v2 = lambda t: color6 if(t == 0)else color2
vfunc = np.vectorize(change_color_v2)
 y=vfunc(y)
 y=eval(f_r)
 return y
<define_variables>
mode='train'
for iteration in range(400):
task = get_data(str(training_path / training_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',training_tasks[iteration] )
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
mode='eval'
for iteration in range(400):
task = get_data(str(evaluation_path / evaluation_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',evaluation_tasks[iteration] )
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
mode='test'
for iteration in range(100):
task = get_data(str(test_path / test_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',test_tasks[iteration] )
<define_variables><EOS> 
pd.options.display.max_rows = 999
display(sample_sub)
 len(sample_sub)
 sample_sub.to_csv('submission.csv' )