Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<count_missing_values>
rc('animation', html='jshtml')
 device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )
test.isnull().sum()<count_missing_values>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge')
 train_path = data_path / 'training'
eval_path = data_path / 'evaluation'
test_path = data_path / 'test'
train_tasks = { task.stem: json.load(task.open())for task in train_path.iterdir() }
eval_tasks = { task.stem: json.load(task.open())for task in eval_path.iterdir() }
test_tasks = { task.stem: json.load(task.open())for task in test_path.iterdir() }
test.isnull().sum()<data_type_conversions>
HIDDEN_SIZE = 128
MAX_STEPS = 10
THRESHOLD = 0.99
REMAINDERS_PEN = 0.0
torch.manual_seed(42)
 np.random.seed(42)
 class CAModel(nn.Module):
def __init__(self, num_states):
super(CAModel, self ).__init__()
self.embedding = nn.Sequential(
 nn.Conv2d(num_states, HIDDEN_SIZE, kernel_size=1),
nn.InstanceNorm2d(HIDDEN_SIZE), 
)
 self.transition = nn.Sequential(
 nn.InstanceNorm2d(HIDDEN_SIZE),
nn.Conv2d(HIDDEN_SIZE, HIDDEN_SIZE, kernel_size=3, padding=1),
nn.ReLU() ,
nn.Conv2d(HIDDEN_SIZE, HIDDEN_SIZE, kernel_size=1, padding=0), 
)
 self.projection_out = nn.Conv2d(HIDDEN_SIZE, num_states, kernel_size=1)
 self.projection_halt = nn.Conv2d(HIDDEN_SIZE, 1, kernel_size=1)
 def forward(self, x, max_steps=None):
x = self.embedding(x)
 halting_probability = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 remainders = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 n_updates = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 max_steps = max_steps or MAX_STEPS
for i in range(max_steps):
p = torch.sigmoid(self.projection_halt(x)- 1)
 still_running =(halting_probability <= THRESHOLD ).to(torch.float)
 new_halted =(( halting_probability + p * still_running)> THRESHOLD ).to(torch.float)* still_running
still_running =(( halting_probability + p * still_running)<= THRESHOLD ).to(torch.float)* still_running
halting_probability += p * still_running
remainders += new_halted *(1 - halting_probability)
 halting_probability += new_halted * remainders
n_updates += still_running + new_halted
update_weights = p * still_running + new_halted * remainders
transformed_state = self.transition(x)
 x = transformed_state * update_weights + x *(1 - update_weights)
 if still_running.sum() == 0:
break
x = self.projection_out(x)
 self._remainders = remainders
self._n_updates = n_updates
return x
test['Fare'].fillna(test['Fare'].quantile(0.5), inplace = True )<count_missing_values>
def solve_task(task, max_steps=10):
model = CAModel(10 ).to(device)
 num_epochs = 100
criterion = nn.CrossEntropyLoss()
losses = np.zeros(num_epochs)
 n_updates = np.zeros(num_epochs)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.001)
 for e in range(num_epochs):
optimizer.zero_grad()
loss = 0.0
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 y = torch.tensor(sample["output"] ).long().unsqueeze(0 ).to(device)
 y_pred = model(x)
 loss += criterion(y_pred, y)+(model._remainders * REMAINDERS_PEN ).mean(0 ).sum()
n_updates[e] += model._n_updates.detach().cpu().mean().numpy() / len(task)
 loss.backward()
optimizer.step()
losses[e] = loss.item()
return model, losses, n_updates
@torch.no_grad()
def predict(model, task):
predictions = []
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 pred = model(x ).argmax(1 ).squeeze().cpu().numpy()
predictions.append(pred)
 return predictions
task = train_tasks["db3e9e38"]["train"]
model, losses, n_updates = solve_task(task )
test.isnull().sum()<count_missing_values>
predictions = predict(model, task)
 for i in range(len(task)) :
plot_sample(task[i], predictions[i] )
test.isnull().sum()<categorify>
test = train_tasks["db3e9e38"]["test"]
predictions = predict(model, test)
 for i in range(len(test)) :
plot_sample(test[i], predictions[i] )
sex1 = pd.get_dummies(train['Sex'])
 sex2 = pd.get_dummies(test['Sex'] )<drop_column>
def evaluate(tasks, is_test=False):
result = []
predictions = {}
for idx, task in tqdm(tasks.items()):
if input_output_shape_is_same(task):
model, _, _ = solve_task(task["train"])
 pred = predict(model, task["test"])
 if not is_test:
score = calk_score(task["test"], pred)
 else:
score = [0] * len(task["test"])
 else:
pred = [el["input"] for el in task["test"]]
score = [0] * len(task["test"])
 predictions[idx] = pred
result.append(score)
 return result, predictions
train.drop('Sex', axis = 1, inplace = True)
 test.drop('Sex', axis = 1, inplace = True)
<concatenate>
train_result, train_predictions = evaluate(train_tasks)
 train_solved = [any(score)for score in train_result]
total = sum([len(score)for score in train_result])
 print(f"solved : {sum(train_solved)} from {total}({sum(train_solved)/total})" )
train = pd.concat([train, sex1], axis=1)
 test = pd.concat([test, sex2], axis=1 )<drop_column>
eval_result, eval_predictions = evaluate(eval_tasks)
 eval_solved = [any(score)for score in eval_result]
total_eval = sum([len(score)for score in eval_result])
 print(f"Eval: solved : {sum(eval_solved)} from {total_eval}({sum(eval_solved)/total})" )
train.drop('female', axis = 1, inplace = True)
 test.drop('female', axis = 1, inplace = True )<categorify>
test_result, test_predictions = evaluate(test_tasks, is_test=True )
Embarked1 = pd.get_dummies(train['Embarked'])
 Embarked2 = pd.get_dummies(test['Embarked'])
 train.drop(['Embarked'], axis = 1, inplace = True)
 test.drop(['Embarked'], axis = 1, inplace = True)
 train = pd.concat([train, Embarked1], axis=1)
 test = pd.concat([test, Embarked2], axis=1 )<feature_engineering>
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )
def family(x):
if x['SibSp'] + x['Parch'] > 1:
return 1
else:
return 0
train['Family'] = train.apply(family, axis=1)
 test['Family'] =test.apply(family, axis = 1 )<drop_column>
def flattener(pred):
str_pred = str([list(row)for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
train.drop(['SibSp','Parch'], axis=1, inplace=True)
 test.drop(['SibSp','Parch'], axis=1, inplace=True)
<feature_engineering>
for output_id in submission.index:
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = str(test_path / str(task_id + '.json'))
with open(f, 'r')as read_file:
task = json.load(read_file)
 data = test_predictions[task_id][pair_id]
pred_1 = flattener(data)
 data = [[5 if i==0 else i for i in j] for j in data]
pred_2 = flattener(data)
 data = [[0 for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv')
 submission.head()