Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<predict_on_test>
import numpy as np
import pandas as pd
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np
model.eval()
predictions = []
for data in tqdm(test_loader):
data = data.to(device)
 output = model(data ).max(dim=1)[1]
predictions += list(output.data.cpu().numpy() )<save_to_csv>
input_path = "/kaggle/input/abstraction-and-reasoning-challenge"
for dirname, _, filenames in os.walk(input_path):
print(dirname )
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission['label'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
data_path = Path(input_path)
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
%matplotlib inline
BATCH_SIZE = 256
print(os.listdir('.. /input/Kannada-MNIST/'))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )<load_from_csv>
task_file = str(training_path / '00d62c1b.json')
 with open(task_file, 'r')as f:
task = json.load(f)
 print(task.keys() )
df_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 target = df_train['label']
df_train.drop('label', axis=1, inplace=True)
 X_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 X_test.drop('id', axis=1, inplace=True)
 X_test.head()<split>
n_train_pairs = len(task['train'])
 n_test_pairs = len(task['test'])
 print(f'task contains {n_train_pairs} training pairs')
 print(f'task contains {n_test_pairs} test pairs' )
X_train, X_dev, y_train, y_dev = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.01)
 print('X_train', len(X_train))
print('X_dev', len(X_dev))
print('X_test', len(X_test))<categorify>
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )
class CharData(Dataset):

 def __init__(self,
images,
labels=None,
transform=None, 
):
self.X = images
self.y = labels
self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
img = np.array(self.X.iloc[idx, :], dtype='uint8' ).reshape([28, 28, 1])
 if self.transform is not None:
img = self.transform(img)
 if self.y is not None:
y = np.zeros(10, dtype='float32')
 y[self.y.iloc[idx]] = 1
return img, y
else:
return img<categorify>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
train_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees=5, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=5),
transforms.ToTensor()
])
 test_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor()
] )<prepare_x_and_y>
example_grid = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
display(example_grid)
 print(flattener(example_grid))
train_dataset = CharData(X_train, y_train, train_transform)
 dev_dataset = CharData(X_dev, y_dev, test_transform)
 test_dataset = CharData(X_test, transform=test_transform )<load_pretrained>
def read_json_file(fileName):
with open(fileName, 'r')as f:
return json.load(f )
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
 test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 )<define_search_model>
def task_train001(df_input):
df_input = np.array(df_input)
 df_output = np.array(df_output)
 color_input, color_output = input_all_colors[0], new_colors[0]
def get_closed_area(arr):
H, W = arr.shape
Dy = [0, -1, 0, 1]
Dx = [1, 0, -1, 0]
arr_padded = np.pad(arr,(( 1, 1),(1, 1)) , "constant", constant_values=0)
 searched = np.zeros(arr_padded.shape, dtype=bool)
 searched[0, 0] = True
q = [(0, 0)]
while q:
y, x = q.pop()
for dy, dx in zip(Dy, Dx):
y_, x_ = y + dy, x + dx
if not 0 <= y_ < H + 2 or not 0 <= x_ < W + 2:
continue
if not searched[y_][x_] and arr_padded[y_][x_] == 0:
q.append(( y_, x_))
searched[y_, x_] = True
res = searched[1:-1, 1:-1]
res |= arr == color_input
return ~res
output = df_input.copy()
output[get_closed_area(df_input)] = color_output
return output
def wrapper_task_train001(df_train, df_test_item):
for df in df_train:
input_all_colors = [x for x in list(np.unique(df['input'])) if x != 0]
output_all_colors = [x for x in list(np.unique(df['output'])) if x != 0]
new_colors = [x for x in output_all_colors if x not in input_all_colors]
assert(len(new_colors)== 1)
 remove_border = False
if len(output_all_colors)- len(input_all_colors)== 0:
remove_border = True
def task_train001(x):
x = np.array(x)
 green, yellow = input_all_colors[0], new_colors[0]
def get_closed_area(arr):
H, W = arr.shape
Dy = [0, -1, 0, 1]
Dx = [1, 0, -1, 0]
arr_padded = np.pad(arr,(( 1, 1),(1, 1)) , "constant", constant_values=0)
 searched = np.zeros(arr_padded.shape, dtype=bool)
 searched[0, 0] = True
q = [(0, 0)]
while q:
y, x = q.pop()
for dy, dx in zip(Dy, Dx):
y_, x_ = y + dy, x + dx
if not 0 <= y_ < H + 2 or not 0 <= x_ < W + 2:
continue
if not searched[y_][x_] and arr_padded[y_][x_] == 0:
q.append(( y_, x_))
searched[y_, x_] = True
res = searched[1:-1, 1:-1]
res |= arr == green
return ~res
y = x.copy()
y[get_closed_area(x)] = yellow
if remove_border:
y[y == input_all_colors[0]] = 0
return y
for df in df_train:
assert(np.array_equal(task_train001(df['input']), df['output'])== True)
 print(np.array_equal(task_train001(df_test_item['input']), df_test_item['output']))
return task_train001(df_test_item['input'] )
DEPTH_MULT = 2
class ConvLayer(nn.Module):
def __init__(self, input_size, output_size, kernel_size=3):
super(ConvLayer, self ).__init__()
self.ops = nn.Sequential(
 nn.Conv2d(input_size, output_size, kernel_size=kernel_size, stride=1, padding=kernel_size//2),
nn.BatchNorm2d(output_size),
nn.ReLU(inplace=True)
)
 def forward(self, x):
return self.ops(x)
 class FCLayer(nn.Module):
def __init__(self, input_size, output_size):
super(FCLayer, self ).__init__()
self.ops = nn.Sequential(
 nn.Linear(input_size, output_size),
nn.BatchNorm1d(output_size),
nn.ReLU(inplace=True)
)
 self.residual = input_size == output_size
def forward(self, x):
if self.residual:
return(self.ops(x)+ x)/ np.sqrt(2)
 return self.ops(x)
 def mixup(x, shuffle, lam, i, j):
if shuffle is not None and lam is not None and i == j:
x = lam * x +(1 - lam)* x[shuffle]
return x
class Net(nn.Module):
def __init__(self, num_classes):
super(Net, self ).__init__()
self.conv1 = ConvLayer(1, DEPTH_MULT * 32)
 self.conv2 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv3 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv4 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv5 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 64)
 self.conv6 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv7 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv8 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv9 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv10 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.mp = nn.MaxPool2d(kernel_size=2, stride=2)
 self.fc1 = FCLayer(DEPTH_MULT * 64 * 7 * 7, DEPTH_MULT * 512)
 self.fc2 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.fc3 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.fc4 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.projection = nn.Linear(DEPTH_MULT * 512, 10)
 def forward(self, x):
if isinstance(x, list):
x, shuffle, lam = x
else:
shuffle = None
lam = None
j = np.random.randint(15)
 x = mixup(x, shuffle, lam, 0, j)
 x = self.conv1(x)
 x = mixup(x, shuffle, lam, 1, j)
 x = self.conv2(x)
 x = mixup(x, shuffle, lam, 2, j)
 x = self.conv3(x)
 x = mixup(x, shuffle, lam, 3, j)
 x = self.conv4(x)
 x = self.mp(x)
 x = mixup(x, shuffle, lam, 4, j)
 x = self.conv5(x)
 x = mixup(x, shuffle, lam, 5, j)
 x = self.conv6(x)
 x = mixup(x, shuffle, lam, 6, j)
 x = self.conv7(x)
 x = mixup(x, shuffle, lam, 7, j)
 x = self.conv8(x)
 x = mixup(x, shuffle, lam, 8, j)
 x = self.conv9(x)
 x = mixup(x, shuffle, lam, 9, j)
 x = self.conv10(x)
 x = self.mp(x)
 x = x.view(x.size(0), -1)
 x = mixup(x, shuffle, lam, 10, j)
 x = self.fc1(x)
 x = mixup(x, shuffle, lam, 11, j)
 x = self.fc2(x)
 x = mixup(x, shuffle, lam, 12, j)
 x = self.fc3(x)
 x = mixup(x, shuffle, lam, 13, j)
 x = self.fc4(x)
 x = mixup(x, shuffle, lam, 14, j)
 x = self.projection(x)
 return x<compute_test_metric>
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 used_k = len(submission)
 for output_id in tqdm_notebook(submission.index[:used_k], total = len(submission.index[:used_k])) :
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 fileName = str(test_path / str(task_id + '.json'))
task = read_json_file(fileName)
 try:
res = wrapper_task_train001(task['train'], task['test'][pair_id])
 pred_1 = flattener(res.tolist())
 print("wrapper_task_train001 success", pred_1)
 except:
data = task['test'][pair_id]['input']
pred_1 = flattener(data)
 pred = pred_1
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv' )