Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<categorify>
warnings.filterwarnings("ignore" )
train_data = pd.get_dummies(train_data, columns=['Pclass'], prefix='P')
 train_data = pd.get_dummies(train_data, columns=['age_group'], prefix='AG')
 train_data = pd.get_dummies(train_data, columns=['Sex'], prefix='S')
 train_data = pd.get_dummies(train_data, columns=['parch_group'], prefix='PA')
 train_data = pd.get_dummies(train_data, columns=['sibsp_group'], prefix='SS')
 train_data = pd.get_dummies(train_data, columns=['cabin_group'], prefix='C')
 train_data.drop(columns=['Age'], inplace=True)
 train_data.drop(columns=['SibSp'], inplace=True)
 train_data.drop(columns=['Parch'], inplace=True)
 train_data.drop(columns=['Cabin'], inplace=True )<count_missing_values>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(train_data.isnull().sum())
 print(len(train_data))<load_from_csv>
def train(task, pred_func):
try:
ok = 0
for i, t in enumerate(task["train"]):
t_in, t_out = np.array(t["input"] ).astype('uint8'), np.array(t["output"] ).astype('uint8')
 sz_out = t_out.shape
if len(inspect.getargspec(pred_func ).args)==1:
t_pred = pred_func(t_in)
 else:
t_pred = pred_func(t_in,sz_out)
 if len(t_out)==len(t_pred):
if len(t_out)==1:
if t_pred==t_out:
ok += 1
elif(t_pred==t_out ).all() :
ok += 1
t_pred = []
if ok==len(task["train"]):
for i, t in enumerate(task["test"]):
t_in = np.array(t["input"] ).astype('uint8')
 if len(inspect.getargspec(pred_func ).args)==1:
t_pred.append(pred_func(t_in))
else:
t_pred.append(pred_func(t_in, sz_out))
return t_pred
else:
return None
except:
return None
test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<filter>
def flattener(pred):
str_pred = str([row for row in pred])
 return str_pred.replace(', ', '' ).replace('[[', '|' ).replace('][', '|' ).replace(']]', '|' )
test_data = test_data.loc[:,['PassengerId', 'Pclass', 'Sex', 'Age','Parch', 'SibSp', 'Cabin']]<categorify>
def crop_min(a):
try:
b = np.bincount(a.flatten() ,minlength=10)
 c = int(np.where(b==np.min(b[np.nonzero(b)])) [0])
 coords = np.argwhere(a==c)
 x_min, y_min = coords.min(axis=0)
 x_max, y_max = coords.max(axis=0)
 return a[x_min:x_max+1, y_min:y_max+1]
except:
return a
test_data = pd.get_dummies(test_data, columns=['Pclass'], prefix='P')
 test_data = pd.get_dummies(test_data, columns=['age_group'], prefix='AG')
 test_data = pd.get_dummies(test_data, columns=['Sex'], prefix='S')
 test_data = pd.get_dummies(test_data, columns=['parch_group'], prefix='PA')
 test_data = pd.get_dummies(test_data, columns=['sibsp_group'], prefix='SS')
 test_data = pd.get_dummies(test_data, columns=['cabin_group'], prefix='C')
 test_data.drop(columns=['Age'], inplace=True)
 test_data.drop(columns=['SibSp'], inplace=True)
 test_data.drop(columns=['Parch'], inplace=True)
 test_data.drop(columns=['Cabin'], inplace=True )<count_missing_values>
def crop_max(a):
try:
b = np.bincount(a.flatten() ,minlength=10)
 b[0] = 255
c = np.argsort(b)[-2]
coords = np.argwhere(a==c)
 x_min, y_min = coords.min(axis=0)
 x_max, y_max = coords.max(axis=0)
 return a[x_min:x_max+1, y_min:y_max+1]
except:
return a
print(test_data.isnull().sum())
 print(len(test_data))<prepare_x_and_y>
def resize_o(a,s):
try:
nx,ny = s[1]//a.shape[1],s[0]//a.shape[0]
return np.repeat(np.repeat(a, nx, axis=0), ny, axis=1)
 except:
return a
<split>
def resize_c(a):
c = np.count_nonzero(np.bincount(a.flatten() ,minlength=10)[1:])
 return np.repeat(np.repeat(a, c, axis=0), c, axis=1 )
y = train_data["Survived"]
features = ['P_1','P_2','P_3','AG_0.0','AG_1.0','AG_2.0','AG_3.0','S_0', 'S_1']
X = train_data.loc[:, features]
X_train, X_test, y_train, y_test = train_test_split(X, y)
<normalization>
def resize_2(a):
return np.repeat(np.repeat(a, 2, axis=0), 2, axis=1 )
ss = StandardScaler()
ss.fit_transform(X_train)
 ss.transform(X_test )<train_model>
def repeat_2(a):
return np.tile(a,a.shape )
model_lr = LogisticRegression(solver='liblinear', max_iter=1000)
 model_lr.fit(X_train, y_train)
 predictions_lr = model_lr.predict(X_test )<compute_test_metric>
task = get_data(str(evaluation_path / evaluation_tasks[325]))
check_p(task, resize_c )
score_train = model_lr.score(X_train, y_train)
 score_test = model_lr.score(X_test, y_test)
 print(score_train, score_test )<find_best_params>
task = get_data(str(evaluation_path / evaluation_tasks[148]))
check_p(task, resize_2 )
thresholds = model_lr.decision_function(X_train)
 fpr, tpr, thresholds = roc_curve(y_train, thresholds )<compute_test_metric>
task = get_data(str(training_path / training_tasks[222]))
check_p(task, resize_o )
print(precision_score(y_test, predictions_lr))
print(recall_score(y_test, predictions_lr))<train_model>
task = get_data(str(evaluation_path / evaluation_tasks[311]))
check_p(task, repeat_2 )
model_knn = KNeighborsClassifier(n_neighbors = 10, p = 1)
 model_knn.fit(X_train, y_train)
 predictions_knn = model_knn.predict(X_test )<compute_train_metric>
task = get_data(str(training_path / training_tasks[288]))
check_p(task, resize_o )
probabilities = model_knn.predict_proba(X_train)
 fpr, tpr, threshold = roc_curve(y_train, probabilities[:, 1] )<compute_test_metric>
task = get_data(str(evaluation_path / evaluation_tasks[251]))
check_p(task, repeat_1 )
score_train = model_knn.score(X_train, y_train)
 score_test = model_knn.score(X_test, y_test)
 print(score_train, score_test )<save_to_csv>
task = get_data(str(evaluation_path / evaluation_tasks[224]))
check_p(task, crop_max )
model = model_knn
X_test = test_data.loc[:, features]
predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id' )
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
for output_id in tqdm(submission.index):
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = test_path / str(task_id + '.json')
 if f.is_file() :
task = get_data(f)
 data = task['test'][pair_id]['input']
pred_1 = flattener(data)
 for oper in ['crop_min','crop_max','resize_o','resize_c','resize_2','repeat_1','repeat_2']:
data = train(task, globals() [oper])
 if data:
pred_1 = flattener(data)
 break
data = task['test'][pair_id]['input']
pred_2 = flattener(data)
 data = [[5 if i==0 else i for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
<define_variables><EOS> 
submission.to_csv('submission.csv' )