Kaggle competition: Abstraction and Reasoning Challenge
def criterion(input, target, size_average=True):

 l = -(target * torch.log(F.softmax(input, dim=1)+ 1e-10)).sum(1)
 if size_average:
l = l.mean()
else:
l = l.sum()
return l<choose_model_class>
class TaskDataset(Dataset):
def __init__(self, path, name, mode='train'):
self.tokenizer = BertTokenizer('vocab.txt')
 self.items = self.__load(path, name, mode)
 def __load(self, path, name, mode):
items = []
task = Task(path, name)
 self.task_width, self.task_height, _ = task.max_size()
max_length = 2 +(self.task_width + 1)* self.task_height
if mode == 'train':
for inp, out in task.get_train() :
inp = self.__tokenize(inp, max_length)
 out = self.__tokenize(out, max_length)
 items.append(( name, inp, out))
if mode == 'valid':
for inp, out in task.get_valid() :
inp = self.__tokenize(inp, max_length)
 out = self.__tokenize(out, max_length)
 items.append(( name, inp, out))
if mode == 'tests':
for inp in task.get_tests() :
inp = self.__tokenize(inp, max_length)
 items.append(( name, inp, inp))
return items
def __tokenize(self, sequ, max_length):
sequ_dict = self.tokenizer.encode_plus(sequ, add_special_tokens=False, max_length=max_length, pad_to_max_length=True)
 return np.array(sequ_dict['input_ids'], dtype=np.int64), np.array(sequ_dict['attention_mask'], dtype=np.int64), np.array(sequ_dict['token_type_ids'], dtype=np.int64),
def __len__(self):
return len(self.items)
 def __getitem__(self, idx):
return self.items[idx]
model = Net(10)
 model = model.to(device)
 n_epochs = 160
optimizer = optim.Adam(model.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=n_epochs // 4, gamma=0.1 )<train_model>
INTERMEDIATE_SIZE = 1024
HIDDEN_SIZE = 1024
VOCAB_SIZE = 16
def create_model_config(w, h):
return {
"architectures": [
"BertForMaskedLM"
],
"attention_probs_dropout_prob": 0.5,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.5,
"hidden_size": HIDDEN_SIZE,
"initializer_range": 0.02,
"intermediate_size": INTERMEDIATE_SIZE,
"max_position_embeddings": 2 +(w + 1)* h,
"num_attention_heads": 8,
"num_hidden_layers": 8,
"type_vocab_size": 1,
"vocab_size": VOCAB_SIZE,
"num_labels": 1
}
class Network(BertPreTrainedModel):
def __init__(self, w, h):
config = BertConfig.from_dict(create_model_config(w, h))
super(Network, self ).__init__(config)
 self.bert = BertModel(config)
 self.fc = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE)
 self.init_weights()
def forward(self, X):
ids = X[0]
msk = X[1]
typ = X[2]
output = self.bert(ids, attention_mask=msk, token_type_ids=typ)
 x = output[0]
x = F.relu(self.fc(x))
x = x.transpose(1, 2)
 return x
def train(epoch, history=None):
model.train()
for batch_idx,(data, target)in enumerate(train_loader):
data = data.to(device)
 target = target.to(device)
 alpha = 2
lam = np.random.beta(alpha, alpha)
 shuffle = torch.randperm(data.shape[0])
 target = lam * target +(1 - lam)* target[shuffle]
optimizer.zero_grad()
output = model([data, shuffle, lam])
 loss = criterion(output, target)
 if history is not None:
history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()
loss.backward()
optimizer.step()
if(batch_idx + 1)% 100 == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLR: {:.6f}\tLoss: {:.6f}'.format(
 epoch,(batch_idx + 1)* len(data), len(train_loader.dataset),
100.*(batch_idx + 1)/ len(train_loader),
optimizer.state_dict() ['param_groups'][0]['lr'],
loss.data))
exp_lr_scheduler.step()
def evaluate(epoch, history=None):
model.eval()
loss = 0
correct = 0
with torch.no_grad() :
for data, target in dev_loader:
data = data.to(device)
 target = target.to(device)
 output = model(data)
 loss += criterion(output, target, size_average=False ).data
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()
loss /= len(dev_loader.dataset)
 accuracy = correct / len(dev_loader.dataset)
 if history is not None:
history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()
history.loc[epoch, 'dev_accuracy'] = accuracy
print('Dev loss: {:.4f}, Dev accuracy: {}/{}({:.3f}%)
 '.format(
 loss, correct, len(dev_loader.dataset),
100.* accuracy))<predict_on_test>
class Summary() :
def __init__(self):
self.epoch = -1
self.loss = None
self.accuracy = None
self.values = None
self.history = []
def register(self, epoch, loss, accuracy, values={}):
values['epoch'] = epoch
values['loss'] = loss
values['accuracy'] = accuracy
self.history.append(values)
 if self.epoch < 0 or accuracy > self.accuracy:
self.epoch = epoch
self.loss = loss
self.accuracy = accuracy
self.values = values
return True
return False
def hash(self):
def r4(v): return "{:6.4f}".format(v)
 return F'{self.epoch:02}-{r4(self.accy ).strip() }'
def __str__(self):
def r4(v): return "{:6.4f}".format(v)
 return F'{self.epoch}\t{r4(self.values["loss"])}\t{r4(self.values["accuracy"])}'
class BaseMetrics() :
def __init__(self):
self.loss = None
self.preds = None
self.targs = None
self.values = None
def begin(self):
self.loss = []
self.preds = []
self.targs = []
self.values = None
def update(self, loss, dz, dy):
self.loss.append(self._get_loss(loss))
self.preds.append(self._get_preds(dz))
self.targs.append(self._get_targets(dy))
def _get_loss(self, loss):
return loss.item()
def _get_preds(self, dz):
return dz.detach().cpu().numpy()
def _get_targets(self, dy):
return dy.detach().cpu().numpy()
def commit(self, epoch):
loss = np.mean(self.loss)
 accuracy = 0.0
self.values = {
'loss': 0.0,
'accuracy': 0.0
}
return accuracy
def __str__(self):
def r4(v): return "{:6.4f}".format(v)
 return F'{r4(self.values["loss"])}\t{r4(self.values["accuracy"])}'
class CrossEntropyMetrics(BaseMetrics):
def _get_preds(self, dz):
dz = dz[:,:,1:]
return dz.detach().cpu().numpy()
def _get_targets(self, dy):
dy = dy[:,1:]
return dy.detach().cpu().numpy()
def commit(self, epoch):
loss = np.mean(self.loss)
 preds = np.concatenate(self.preds, axis=0)
 targs = np.concatenate(self.targs, axis=0)
 preds = preds.argmax(axis=1)
 targs = targs
preds = preds.reshape(preds.shape[0], -1)
 targs = targs.reshape(targs.shape[0], -1)
 accuracy =(sum(preds == targs)/ len(preds)).mean()
match = 0
for pred, targ in zip(preds, targs):
if sum(pred != targ)== 0:
match += 1
accuracy += match
self.values = {
'loss': loss,
'accuracy': accuracy
}
return accuracy
model.eval()
predictions = []
for data in tqdm(test_loader):
data = data.to(device)
 output = model(data ).max(dim=1)[1]
predictions += list(output.data.cpu().numpy() )<save_to_csv>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def random_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission['label'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 PATH = '.. /input/abstraction-and-reasoning-challenge'
PATH_TESTS = PATH + '/test'
EPOCHS = 100
BATCH = 4
LR = 0.0001
SEED = 1234
random_seed(SEED)
 TOKENS = ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '[
with open('vocab.txt', 'w')as fp:
for t in TOKENS:
fp.write(t + '
')
 def train(name, epochs=EPOCHS):
ds_train = TaskDataset(PATH_TESTS, name, mode='train')
 ld_train = torch.utils.data.DataLoader(ds_train, batch_size=BATCH, num_workers=4, shuffle=True)
 w = ds_train.task_width
h = ds_train.task_height
model = Network(w, h)
 model = model.to(DEVICE)
 criterion = nn.CrossEntropyLoss()
optimizer = op.Adam(model.parameters() , lr=LR)
 metrics = CrossEntropyMetrics()
summary_train = Summary()
for epoch in range(epochs):
model.train()
metrics.begin()
for step,(id, x, y)in enumerate(ld_train):
optimizer.zero_grad()
x0 = x[0].to(DEVICE)
 x1 = x[1].to(DEVICE)
 x2 = x[2].to(DEVICE)
 dy = y[0].to(DEVICE)
 dz = model(( x0, x1, x2))
loss = criterion(dz[:,:,1:], dy[:,1:])
 metrics.update(loss, dz, dy)
 loss.backward()
optimizer.step()
accuracy = metrics.commit(epoch)
 summary_train.register(epoch, loss.item() , accuracy, metrics.values)
 print(summary_train)
 return model
def predict(model, name):
ds_tests = TaskDataset(PATH_TESTS, name, mode='tests')
 ld_tests = torch.utils.data.DataLoader(ds_tests, batch_size=1, shuffle=False)
 w = ds_tests.task_width
h = ds_tests.task_height
preds = []
model.eval()
with torch.no_grad() :
for step,(id, x, y)in enumerate(ld_tests):
x0 = x[0].to(DEVICE)
 x1 = x[1].to(DEVICE)
 x2 = x[2].to(DEVICE)
 dz = model(( x0, x1, x2))
z = dz.detach().cpu().numpy()
z = z[0]
az = z.argmax(0)
 w, h = Task.pred_size(az)
 w = max(1, w)
 h = max(1, h)
 pred = Task.as_array(az, w, h)
 pred = Task.as_list(pred)
 flat = flattener(pred)
 preds.append(flat)
 return preds
submission = pd.read_csv(PATH + '/sample_submission.csv', index_col='output_id')
 STEPS = -1
step = 0
hash = set()
for output_id in submission.index:
task_id = output_id.split('_')[0]
if task_id in hash:
continue
hash.add(task_id)
 name = str(task_id + '.json')
 model = train(name, EPOCHS)
 preds = predict(model, name)
 for n in range(len(preds)) :
pred = preds[n] + ' ' + preds[n] + ' ' + preds[n]
submission.loc[F'{task_id}_{n}', 'output'] = pred
step += 1
if step == STEPS: break
submission.to_csv('submission.csv')
<load_from_csv><EOS> 
df = pd.read_csv('submission.csv')
 df.head()
tokenizer = BertTokenizer('vocab.txt')
 def tokenize(sequ, max_length):
sequ_dict = tokenizer.encode_plus(sequ, add_special_tokens=False, max_length=max_length, pad_to_max_length=True)
 return np.array(sequ_dict['input_ids'], dtype=np.int64)
 for values in df.values:
name = values[0]
name = name[:-2] + '.json'
print('TRAIN', name)
 task = Task(PATH_TESTS, name)
 for inp, out in task.get_train() :
task_width, task_height, _ = task.max_size()
max_length = 2 +(task_width + 1)* task_height
x = tokenize(inp, max_length)
 y = tokenize(out, max_length)
 wx, hx = Task.pred_size(x)
 arr_x = Task.as_array(x, wx, hx)
 wy, hy = Task.pred_size(y)
 arr_y = Task.as_array(y, wy, hy)
 plot_task_array([arr_x, arr_y])
 ix = 0
print('PREDS', name)
 for inp in task.get_tests() :
try:
x = tokenize(inp, max_length)
 wx, hx = Task.pred_size(x)
 arr_x = Task.as_array(x, wx, hx)
 parts = values[1].split(' ')[0].split('|')
 parts = parts[1:-1]
h = len(parts)
 w = len(parts[0])
 a = np.zeros(( h, w), dtype='int')
 for y in range(h):
for x in range(w):
a[y][x] = parts[y][x]
plot_task_array([arr_x, a])
 except Exception as ex:
print(ex)
 print()