Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<load_from_csv>
import os
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from tqdm import tqdm
train_id = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test_id = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 dig_id = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )<define_variables>
def get_input_matrix(json_file: dict, index: int)-> np.ndarray:
return np.array(json_file['train'][index]['input'])
 def get_input_matrices(json_file: dict)-> list:

 res = []
count_of_train_samples = len(json_file['train'])
 for index in range(count_of_train_samples):
res.append(get_input_matrix(json_file, index))
return res
def get_output_matrix(json_file: dict, index: int)-> np.ndarray:
return np.array(json_file['train'][index]['output'])
 def get_output_matrices(json_file: dict):

 res = []
count_of_test_samples = len(json_file['train'])
 for index in range(count_of_test_samples):
res.append(get_output_matrix(json_file, index))
return res
def get_input_matrix_test(json_file: dict, index: int)-> np.ndarray:

 return np.array(json_file['test'][index]['input'])
 def get_output_matrix_test(json_file: dict, index: int)-> np.ndarray:

 return np.array(json_file['test'][index]['output'])
 def get_input_matrices_test(json_file: dict)-> list:

 res = []
count_of_test_samples = len(json_file['test'])
 for index in range(count_of_test_samples):
res.append(get_input_matrix_test(json_file, index))
return res
def get_output_matrices_test(json_file: dict):

 res = []
count_of_test_samples = len(json_file['test'])
 for index in range(count_of_test_samples):
res.append(get_output_matrix_test(json_file, index))
return res
def load_task(json_file_path: str)-> tuple:

 with open(json_file_path, 'r')as jfile:
json_file = json.load(jfile)
 train_input_matrices = get_input_matrices(json_file)
 train_output_matrices = get_output_matrices(json_file)
 test_input_matrices = get_input_matrices_test(json_file)
 return(train_input_matrices, train_output_matrices, test_input_matrices)
 cmap = colors.ListedColormap(
 ['
'
norm = colors.Normalize(vmin=0, vmax=9)
 def plot_matrix(matrix, subplot_num = 111, title = ''):
ax = plt.subplot(subplot_num)
 plt.title(title)
 ax.imshow(matrix, cmap = cmap, norm = norm)
 ax.grid(True, which = 'both', color = 'black', linewidth = 0.7)
 ax.set_yticks([x - 0.5 for x in range(len(matrix)+ 1)])
 ax.set_xticks([x - 0.5 for x in range(len(matrix[0])+ 1)])
 ax.set_xticklabels([])
 ax.set_yticklabels([])
 def make_output(matrix: np.ndarray)-> str:

 if(matrix is None):
return '||'
rows = ''
for row in matrix:
rows += str(row)
 rows = rows.replace(' ', '')
 rows = rows.replace('][', '|')
 rows = rows.replace('[[', '|')
 rows = rows.replace('[', '|')
 rows = rows.replace(']]', '|')
 rows = rows.replace(']', '|')
 return rows
def check_size_1piece(input_matrices: list, output_matrices: list)-> bool:

 for inp, out in zip(input_matrices, output_matrices):
vertical_ratio = out.shape[0] / inp.shape[0]
horizontal_ration = out.shape[1] / inp.shape[1]
cond1 = vertical_ratio == horizontal_ration and vertical_ratio == 1
if not cond1:
return False
return True
x, y, z = 28, 28, 1
qtd_classes = 10
images = []
images_labels = []
valid = []
valid_labels = []
for count,(index, row)in enumerate(train_id.iterrows()):
images.append(row.values[1:].reshape(x, y, z))
images_labels.append(row.values[:1])
 for count,(index, row)in enumerate(dig_id.iterrows()):
valid.append(row.values[1:].reshape(x, y, z))
valid_labels.append(row.values[:1] )<concatenate>
def is_similar(arr1: np.ndarray, arr2: np.ndarray, extra_color: int, min_possible_elements = 9)-> bool:

 if(arr1.size != arr2.size or arr1.size == 0 or arr1.ndim != 1 or arr2.ndim != 1):
return False
if(np.any(arr1 == extra_color)and np.any(arr2 == extra_color)) :
return False
count_equal = 0
for i in range(arr1.shape[0]):
if(arr1[i] == extra_color or arr2[i] == extra_color):
continue
if(arr1[i] != arr2[i]):
return False
count_equal += 1
return count_equal >= min_possible_elements
TESTER = []
for count,(index, row)in enumerate(test_id.iterrows()):
TESTER.append(row.values[1:].reshape(x, y, z))<categorify>
def find_equal_segment(inp: np.ndarray, bad_segment: np.ndarray, extra_color: int)-> np.ndarray:

 segment_len = bad_segment.size
for i in range(inp.shape[0]):
for col in range(inp.shape[1] - segment_len + 1):
current_segment = inp[i, col:col + segment_len]
if(is_similar(current_segment, bad_segment, extra_color)) :
return current_segment
for j in range(inp.shape[1]):
for row in range(inp.shape[0] - segment_len + 1):
current_segment = inp[row:row + segment_len, j]
if(is_similar(current_segment, bad_segment, extra_color)) :
return current_segment
return None
SUBMIT = np.asarray(TESTER)/ 255.
 TEST = np.asarray(valid)/ 255.
 TRAIN = np.asarray(images)/ 255.
 TRAIN_labels = to_categorical(np.asarray(images_labels), qtd_classes)
 TEST_labels = to_categorical(np.asarray(valid_labels), qtd_classes )<concatenate>
def delete_extra_color(inp: np.ndarray, extra_color: int):

 bad_rows, bad_cols = np.where(inp == extra_color)
 bad_rows = np.unique(bad_rows)
 bad_cols = np.unique(bad_cols)
 if(len(bad_rows)> inp.shape[0] // 2 or len(bad_cols)> inp.shape[0] // 2):
return
for i in bad_rows:
for segment_len in range(10, inp.shape[1] + 1, 1):
for col in range(inp.shape[1] - segment_len + 1):
bad_segment = inp[i, col: col + segment_len]
if(not np.any(bad_segment == extra_color)) :
continue
correct_segment = find_equal_segment(inp, bad_segment, extra_color)
 if(correct_segment is not None):
inp[i, col:col + segment_len] = correct_segment
for j in bad_cols:
for segment_len in range(10, inp.shape[0] + 1, 1):
for row in range(inp.shape[0] - segment_len + 1):
bad_segment = inp[row:row + segment_len, j]
if(not np.any(bad_segment == extra_color)) :
continue
correct_segment = find_equal_segment(inp, bad_segment, extra_color)
 if(correct_segment is not None):
inp[row:row + segment_len, j] = correct_segment
train = np.concatenate([TEST, TRAIN])
 labels = np.concatenate([TRAIN_labels, TEST_labels] )<split>
def prediction(inp: np.ndarray,
symmetric_axis_horizontal: list,
symmetric_axis_vertical: list,
symmetric_main_diagonal: bool,
symmetric_side_diagonal: bool,
shift_i: int,
shift_j: int,
size: int,
extra_color: int)-> bool:

 res = inp.copy()
for i in range(inp.shape[0]):
for j in range(inp.shape[1]):
if(inp[i][j] != extra_color):
res[i][j] = inp[i][j]
continue
if symmetric_axis_horizontal is not None:
if(symmetric_axis_horizontal[1] != 0):
vertical_len = min(abs(i - symmetric_axis_horizontal[0]), abs(i - symmetric_axis_horizontal[1]))
row = i + vertical_len * 2 + 1
if(i >= symmetric_axis_horizontal[1]):
row = i - vertical_len * 2 - 1
if(0 <= row and row < inp.shape[0] and inp[row][j] != extra_color):
res[i][j] = inp[row][j]
else:
vertical_len = abs(i - symmetric_axis_horizontal[0])
 row = i + vertical_len * 2
if(i >= symmetric_axis_horizontal[0]):
row = i - vertical_len * 2
if(0 <= row and row < inp.shape[0] and inp[row][j] != extra_color):
res[i][j] = inp[row][j]
if symmetric_axis_vertical is not None:
if(symmetric_axis_vertical[1] != 0):
horizontal_len = min(abs(j - symmetric_axis_vertical[0]), abs(j - symmetric_axis_vertical[1]))
column = j + horizontal_len * 2 + 1
if(j >= symmetric_axis_vertical[1]):
column = j - horizontal_len * 2 - 1
if(0 <= column and column < inp.shape[1] and inp[i][column] != extra_color):
res[i][j] = inp[i][column]
else:
horizontal_len = abs(j - symmetric_axis_vertical[0])
 column = j + horizontal_len * 2
if(j >= symmetric_axis_vertical[0]):
column = j - horizontal_len * 2
if(0 <= column and column < inp.shape[1] and inp[i][column] != extra_color):
res[i][j] = inp[i][column]
if(symmetric_main_diagonal):
if i - shift_i > j - shift_j:
new_i = j + abs(shift_j - shift_i)
 new_j = i - abs(shift_j - shift_i)
 elif i - shift_i < j - shift_j:
new_i = j - abs(shift_j - shift_i)
 new_j = i + abs(shift_j - shift_i)
 else:
new_i, new_j = -1, -1
if(0 <= new_i and new_i < inp.shape[0] and 0 <= new_j and new_j < inp.shape[1] and inp[new_i][new_j] != extra_color):
res[i][j] = inp[new_i][new_j]
if(symmetric_side_diagonal):
side_diag_sum = shift_i + shift_j + size - 1
delta = side_diag_sum - i - j
if(delta % 2 == 0):
if(delta > 0):
delta += 1
elif(delta < 0):
delta -= 1
new_i = i + delta
new_j = j + delta
if(0 <= new_i and new_i < inp.shape[0] and 0 <= new_j and new_j < inp.shape[1] and
inp[new_i][new_j] != extra_color):
res[i][j] = inp[new_i][new_j]
if(np.any(res == extra_color)) :
delete_extra_color(res, extra_color)
 return res
x_train, x_test, y_train, y_test = train_test_split(train, labels, test_size = 0.3 )<define_variables>
def find_restored_matrix(inp: np.ndarray, out: np.ndarray, mode = 'train')-> list:

 min_possible_size = max(11, inp.shape[0] // 2)
 for size in range(min_possible_size, inp.shape[0] + 1, 1):
symmetric_axis_horizontal = None
symmetric_axis_vertical = None
for i in range(inp.shape[0] - size + 1):
for j in range(inp.shape[1] - size + 1):
sub_matrix = inp[i:i + size, j:j + size].copy()
result_horizontal = is_symmetric(sub_matrix, 1)
 result_vertical = is_symmetric(sub_matrix, 2)
 result_main_diag = is_symmetric(sub_matrix, 3)
 result_side_diag = is_symmetric(sub_matrix, 4)
 if(result_horizontal[0]):
extra_color = result_horizontal[1]
elif(result_vertical[0]):
extra_color = result_vertical[1]
elif(result_main_diag[0]):
extra_color = result_main_diag[1]
elif(result_side_diag[0]):
extra_color = result_side_diag[1]
else:
break
if(result_horizontal[0] and extra_color != result_horizontal[1]):
break
if(result_vertical[0] and extra_color != result_vertical[1]):
break
if(result_main_diag[0] and extra_color != result_main_diag[1]):
break
if(result_side_diag[0] and extra_color != result_side_diag[1]):
break
if(size % 2 == 0):
vertical_index = [0, 0]
vertical_index[0] =(j + j + size)// 2 - 1
vertical_index[1] =(j + j + size)// 2
horizontal_index = [0, 0]
horizontal_index[0] =(i + i + size)// 2 - 1
horizontal_index[1] =(i + i + size)// 2
if(result_horizontal[0]):
symmetric_axis_horizontal = horizontal_index
if(result_vertical[0]):
symmetric_axis_vertical = vertical_index
else:
vertical_index =(j + j + size)// 2
horizontal_index =(i + i + size)// 2
if(result_horizontal[0]):
symmetric_axis_horizontal = [horizontal_index, 0]
if(result_vertical[0]):
symmetric_axis_vertical = [vertical_index, 0]
cur = prediction(inp, symmetric_axis_horizontal, symmetric_axis_vertical,
result_main_diag[0], result_side_diag[0], i, j, size, extra_color)
 if(np.any(cur == extra_color)) :
continue
if(mode == 'train' and np.array_equal(cur, out)) :
return cur
elif(mode == 'test'):
return cur
return None
NUM_CLASSES = 10
EPOCHS = 15
BATCH_SIZE = 64
inputShape =(x, y, z )<choose_model_class>
def symmetry_algo(train_in: list, train_out: list, test_in: list)-> list:

 for inp, out in zip(train_in, train_out):
if(inp.shape[0] < 11):
return None
predict = find_restored_matrix(inp, out)
 if(predict is None):
return None
result = []
for test_inp in test_in:
if(test_inp.shape[0] < 11):
return None
predict = find_restored_matrix(test_inp, inp, mode = 'test')
 if(predict is None):
return None
result.append(predict)
 return result
<compute_test_metric><EOS> 
data_folder = '.. /input/abstraction-and-reasoning-challenge/evaluation/'
results = pd.DataFrame(columns =('output_id', 'output'))
mode = 'Train'
correct_predictions = 0
total_predictions = 0
for filename in tqdm(os.listdir(data_folder)) :
full_path = os.path.join(data_folder, filename)
 train_input_matrices, train_output_matrices, test_input_matrices = load_task(full_path)
 if(mode == 'Train'):
with open(full_path, 'r')as jfile:
json_file = json.load(jfile)
 test_output_matrices = get_output_matrices_test(json_file)
 solution_found = False
if(check_size_1piece(train_input_matrices, train_output_matrices)) :
predictions = symmetry_algo(train_input_matrices, train_output_matrices, test_input_matrices)
 total_predictions += len(test_input_matrices)
 if predictions is not None:
print('Found for: ', filename)
 solution_found = True
for ind, test_out in enumerate(predictions):
filename = filename.split('.')[0]
results = results.append({'output_id': filename + '_' + str(ind), 'output': make_output(test_out)}, ignore_index = True)
 if(mode == 'Train'):
if(np.array_equal(test_out, test_output_matrices[ind])) :
correct_predictions += 1
plt.figure(figsize =(10, 5), dpi = 150)
 plot_matrix(test_input_matrices[ind], 131, 'Test input matrix')
 plot_matrix(test_output_matrices[ind], 132, 'Test output matrix')
 plot_matrix(test_out, 133, 'Test predicted matrix')
 plt.show()
if(not solution_found):
predictions = np.array([])
 for ind in range(len(test_input_matrices)) :
results = results.append({'output_id': filename.split('.')[0] + '_' + str(ind), 'output': make_output(predictions)}, ignore_index = True)
 if(mode == 'Train'):
print(f'Solved {correct_predictions} of {total_predictions}')
 results.to_csv('submission.csv', index = False )