Kaggle competition: Abstraction and Reasoning Challenge
train['Cabin'] = pd.Series(i[0] if not pd.isnull(i)else 'X' for i in train['Cabin'])
 test['Cabin'] = pd.Series(i[0] if not pd.isnull(i)else 'X' for i in test['Cabin'] )<categorify>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(len(training_tasks), len(evaluation_tasks), len(test_tasks))
train['Cabin'] = train['Cabin'].map({
'X': 0,
'A': 1,
'B': 2,
'C': 3,
'D': 4,
'E': 5,
'F': 6,
'G': 7,
'T': 0
})
 train['Cabin'] = train['Cabin'].astype(int)
 test['Cabin'] = test['Cabin'].map({
'X': 0,
'A': 1,
'B': 2,
'C': 3,
'D': 4,
'E': 5,
'F': 6,
'G': 7,
'T': 0
})
 test['Cabin'] = test['Cabin'].astype(int )<feature_engineering>
def get_data(task_filename):
with open(task_filename, 'r')as f:
task = json.load(f)
 return task
num2color = ["black", "blue", "red", "green", "yellow", "gray", "magenta", "orange", "sky", "brown"]
color2num = {c: n for n, c in enumerate(num2color)}
train_title = [i.split(",")[1].split(".")[0].strip() for i in train["Name"]]
train["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title )<drop_column>
def task_train000(x):
x_upsampled = x.repeat(3, axis=0 ).repeat(3, axis=1)
 x_tiled = np.tile(x,(3, 3))
y = x_upsampled & x_tiled
return y
task = get_data(str(training_path / training_tasks[0]))
check(task, task_train000 )
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )<categorify>
df_train = pd.DataFrame(training_tasks, columns=['id'])
 df_train['id'] = df_train['id'].apply(lambda x: x[:-5])
 df_train.head()
train["Title"] = train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train["Title"] = train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train["Title"] = train["Title"].astype(int)
 test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )<feature_engineering>
def get_num_train(x):
data = get_data("{}/{}.json".format(training_path, x))
ans = len(data['train'])
 return ans
df_train['num_train'] = -1
df_train['num_train'] = df_train['id'].apply(lambda x: get_num_train(x))
df_train.head()
Ticket1 = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train["Ticket"] = Ticket1
Ticket2 = []
for j in list(test.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test["Ticket"] = Ticket2<categorify>
def get_num_test(x):
data = get_data("{}/{}.json".format(training_path, x))
ans = len(data['test'])
 return ans
df_train['num_test'] = -1
df_train['num_test'] = df_train['id'].apply(lambda x: get_num_test(x))
df_train.head()
train= pd.get_dummies(train, columns = ["Ticket"], prefix="T")
 test = pd.get_dummies(test, columns = ["Ticket"], prefix="T" )<drop_column>
def get_inputshape_is_output_shape(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if input_array.shape!=output_array.shape:
ans = False
return ans
df_train['inputshape_is_output_shape'] = -1
df_train['inputshape_is_output_shape'] = df_train['id'].apply(lambda x: get_inputshape_is_output_shape(x))
df_train.head()
train = train.drop(['T_SP','T_SOP','T_Fa','T_LINE','T_SWPP','T_SCOW','T_PPP','T_AS','T_CASOTON'],axis = 1)
 test = test.drop(['T_SCA3','T_STONOQ','T_AQ4','T_A','T_LP','T_AQ3'],axis = 1 )<drop_column>
def get_inputshape_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
input_shape = np.array(data['train'][0]['input'] ).shape
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if input_array.shape!=input_shape:
ans = False
return ans
df_train['inputshape_is_same'] = -1
df_train['inputshape_is_same'] = df_train['id'].apply(lambda x: get_inputshape_is_same(x))
df_train.head()
train.drop(['Survived'],axis=1,inplace=True )<count_missing_values>
def get_outputshape_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
output_shape = np.array(data['train'][0]['output'] ).shape
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if output_array.shape!=output_shape:
ans = False
return ans
df_train['outputshape_is_same'] = -1
df_train['outputshape_is_same'] = df_train['id'].apply(lambda x: get_outputshape_is_same(x))
df_train.head()
train.isnull().sum() 
<count_missing_values>
def get_inputunique_is_outputunique(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(input_array ).shape!=np.unique(output_array ).shape:
ans = False
else:
for j in range(len(np.unique(input_array))):
if np.unique(input_array)[j]!=np.unique(output_array)[j]:
ans = False
return ans
df_train['inputunique_is_outputunique'] = -1
df_train['inputunique_is_outputunique'] = df_train['id'].apply(lambda x: get_inputunique_is_outputunique(x))
df_train.head()
test.isnull().sum()<count_missing_values>
def get_inputunique_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
input_unique = np.unique(np.array(data['train'][0]['input']))
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(input_array ).shape!=input_unique.shape:
ans = False
else:
for j in range(len(input_unique)) :
if np.unique(input_array)[j]!=input_unique[j]:
ans = False
return ans
df_train['inputunique_is_same'] = -1
df_train['inputunique_is_same'] = df_train['id'].apply(lambda x: get_inputunique_is_same(x))
df_train.head()
test.isnull().sum()<normalization>
def get_outputunique_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
output_unique = np.unique(np.array(data['train'][0]['output']))
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(output_array ).shape!=output_unique.shape:
ans = False
else:
for j in range(len(output_unique)) :
if np.unique(output_array)[j]!=output_unique[j]:
ans = False
return ans
df_train['outputunique_is_same'] = -1
df_train['outputunique_is_same'] = df_train['id'].apply(lambda x: get_outputunique_is_same(x))
df_train.head()
scaler = StandardScaler()
train2 = scaler.fit_transform(train)
 test2 = scaler.fit_transform(test )<create_dataframe>
tmp = df_train.iloc[:,3:].values==df_train.iloc[0,3:].values
tmp = tmp.min(axis=1)
 tmp
KFold_Score = pd.DataFrame()
classifiers = ['Linear SVM', 'Radial SVM', 'LogisticRegression',
'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier', 'KNeighborsClassifier','GradientBoostingClassifier']
models = [svm.SVC(kernel='linear'),
svm.SVC(kernel='rbf'),
LogisticRegression(max_iter = 1000),
RandomForestClassifier(n_estimators=200, random_state=0),
AdaBoostClassifier(random_state = 0),
xgb.XGBClassifier(n_estimators=100),
KNeighborsClassifier() ,
GradientBoostingClassifier(random_state=0)
 ]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=0, shuffle=True)
 KFold_Score[classifiers[j]] =(cross_val_score(model, train2, np.ravel(pred), scoring = 'accuracy', cv=cv))
j = j+1<create_dataframe>
tmp = df_train.iloc[:,3:].values==df_train.iloc[0,3:].values
tmp = tmp.min(axis=1)
 df_tmp = df_train[tmp]
df_tmp
mean = pd.DataFrame(KFold_Score.mean() , index= classifiers)
 KFold_Score = pd.concat([KFold_Score,mean.T])
 KFold_Score.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
KFold_Score.T.sort_values(by=['Mean'], ascending = False )<feature_engineering>
def copy_horizontal(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred], axis=1)
 return tmp
idx = 56
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, crop0)
 data_idx = proc_base(data_idx, copy_horizontal)
 plot_task2(data_idx )
col_name1[0],col_name1[2] = col_name1[2],col_name1[0]
col_name2[0],col_name2[2] = col_name2[2],col_name2[0]<prepare_x_and_y>
def copy_flip(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[:,::-1]], axis=1)
 return tmp
def copy_flop(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[::-1]], axis=0)
 return tmp
idx = 82
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, copy_flip)
 data_idx = proc_base(data_idx, copy_flop)
 plot_task2(data_idx )
train_new = train[col_name1]
test_new = test[col_name2]<drop_column>
def flip(t_in, t_out, t_pred):
tmp = t_pred[:,::-1]
return tmp
def flop(t_in, t_out, t_pred):
tmp = t_pred[::-1]
return tmp
idx = 115
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, flop)
 data_idx = proc_base(data_idx, copy_flop)
 plot_task2(data_idx )
train_new = train_new.drop(['Cabin'],axis = 1)
 test_new = test_new.drop(['Cabin'],axis = 1 )<normalization>
def copy_rotate90(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred.T[:,::-1]], axis=1)
 return tmp
def rotate90(t_in, t_out, t_pred):
tmp = t_pred.T[:,::-1]
return tmp
def rotate270(t_in, t_out, t_pred):
tmp = t_pred.T[::-1]
return tmp
def copy_rotate180(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[::-1,::-1]], axis=1)
 return tmp
idx = 193
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, copy_rotate90)
 data_idx = proc_base(data_idx, rotate270)
 plot_task2(data_idx )
sc = StandardScaler()
train3 = sc.fit_transform(train_new)
 test3 = sc.transform(test_new )<choose_model_class>
def proc_base(data, func):
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
t_pred = data['train'][i]['predict']
try:
data['train'][i]['predict'] = func(t_in, t_out, t_pred)
 except:
data['train'][i]['predict'] = t_pred
for i in range(len(data['test'])) :
t_in = data['test'][i]['input']
t_out = data['train'][0]['output']
t_pred = data['test'][i]['predict']
try:
data['test'][i]['predict'] = func(t_in, t_out, t_pred)
 except:
data['test'][i]['predict'] = t_pred
return data
def tile_reflect(t_in, t_out, t_pred):
tmp = np.zeros_like(t_out)
 h, w = t_out.shape[0]//t_in.shape[0], t_out.shape[1]//t_in.shape[1]
for i in range(h):
for j in range(w):
tile = t_pred
if i%2==1: tile = tile[::-1]
if j%2==1: tile = tile[:, ::-1]
tmp[i*t_in.shape[0]:(i+1)*t_in.shape[0], j*t_in.shape[1]:(j+1)*t_in.shape[1]] = tile
return tmp
def rotate180(t_in, t_out, t_pred):
tmp = t_pred[::-1, ::-1]
return tmp
idx = 210
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, rotate180)
 data_idx = proc_base(data_idx, tile_reflect)
 plot_task2(data_idx )
clf = RandomForestClassifier(random_state=0)
 param_grid={
'n_estimators': [200,300],
'max_features': ['auto', 'sqrt'],
'max_depth': [6,7,8],
'criterion':['gini','entropy']
}<train_on_grid>
func_list = [
func_resize,
func_tile0,
slice4,
crop0,
copy_horizontal,
copy_flip,
copy_flop,
flip,
flop,
copy_rotate90,
rotate90,
rotate270,
copy_rotate180,
tile_reflect,
rotate180,
]
CV_clf = GridSearchCV(estimator=clf, param_grid=param_grid, cv=5)
 CV_clf.fit(train3, pred)
 CV_clf.best_params_<train_model>
def judge(data):
ans = True
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
t_pred = data['train'][i]['predict']
if t_out.shape!=t_pred.shape: ans = False
elif(t_out==t_pred ).min() ==0: ans = False
return ans
idx = 210
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 print(judge(data_idx))
data_idx = proc_base(data_idx, rotate180)
 data_idx = proc_base(data_idx, tile_reflect)
 print(judge(data_idx))
plot_task2(data_idx )
clf1 = RandomForestClassifier(random_state=0, n_estimators=200, criterion='gini', max_features='auto', max_depth=8)
 clf1.fit(train3, pred )<predict_on_test>
import copy
pred3 = clf1.predict(test3 )<save_to_csv>
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 for i in range(1000):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, 2)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
plot_task2(data_tmp )
pred_test = pred3
output = pd.DataFrame({
'PassengerId': test_data.PassengerId,
'Survived': pred_test
})
 output.to_csv('submission1.csv', index=False )<load_from_csv>
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
plot_task2(data_tmp )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data<feature_engineering>
idx = np.random.randint(len(df_train))
print(idx)
 data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
print(func_tmp)
 plot_task2(data_tmp )
t = train_data.loc[:, ['Cabin', 'Survived', 'PassengerId']]
t['initial'] = t['Cabin'].str[0]
t.groupby(['initial', 'Survived'] ).agg('count' )<filter>
import copy
train_data = train_data.loc[:,['PassengerId', 'Survived', 'Pclass', 'Sex', 'Age', 'Parch', 'SibSp', 'Cabin']]
train_data<groupby>
def proc_init(data):
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
data['train'][i]['input'] = np.array(t_in)
 data['train'][i]['output'] = np.array(t_out)
 data['train'][i]['predict'] = np.array(t_in)
 rate_h = copy.deepcopy(data['train'][i]['output'].shape[0]/data['train'][i]['input'].shape[0])
 rate_w = copy.deepcopy(data['train'][i]['output'].shape[1]/data['train'][i]['input'].shape[1])
 for i in range(len(data['test'])) :
t_in = np.array(data['test'][i]['input'])
 t_out = np.zeros([int(t_in.shape[0]*rate_h),int(t_in.shape[1]*rate_w)], dtype=t_in.dtype)
 data['test'][i]['input'] = t_in
data['test'][i]['output'] = t_out
data['test'][i]['predict'] = np.array(t_in)
 return data
count = 0
for idx in range(len(df_train.iloc[:])) :
print(idx)
 data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),
]
flag = False
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 print(func_tmp)
 plot_task2(data_tmp)
 flag = True
count += 1
break
if flag: break
print(count )
train_data.loc[:, ['Parch', 'Survived', 'PassengerId']].groupby(['Parch', 'Survived'] ).agg('count' )<feature_engineering>
def flattener(pred):
str_pred = str([row for row in pred.tolist() ])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
t = train_data.loc[:, ['Age', 'Survived', 'PassengerId']]
t['age_group'] = t['Age'] // 10
t.groupby(['age_group', 'Survived'] ).agg('count' )<groupby>
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
flag = False
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 flag = True
break
if flag: break
if flag: break
print(data_tmp['test'][0]['predict'])
 print(flattener(data_tmp['test'][0]['predict']))
plot_task2(data_tmp )
train_data.loc[:, ['SibSp', 'Survived', 'PassengerId']].groupby(['SibSp', 'Survived'] ).agg('count' )<groupby>
df_sub = pd.read_csv(".. /input/abstraction-and-reasoning-challenge/sample_submission.csv")
 df_sub.head()
sex_age_ave = train_data.loc[:, ['Sex', 'Age']].groupby(['Sex'] ).agg({'Age':'mean'})
 male_age_ave = sex_age_ave.loc['male'].values[0]
female_age_ave = sex_age_ave.loc['female'].values[0]
print(male_age_ave, female_age_ave )<feature_engineering>
df_test = pd.DataFrame(test_tasks, columns=['id'])
 df_test['id'] = df_test['id'].apply(lambda x: x[:-5])
 df_test.head()
def create_age_group(df):
df.loc[(df['Age'] >= 0)&(df['Age'] < 15), ['age_group']] = 0
df.loc[(df['Age'] >= 15)&(df['Age'] < 25), ['age_group']] = 1
df.loc[(df['Age'] >= 25)&(df['Age'] < 65), ['age_group']] = 2
df.loc[(df['Age'] >= 65), ['age_group']] = 3<feature_engineering>
def record_answer(idx, df_sub, data):
for i in range(len(data['test'])) :
t_in = data['test'][i]['input']
t_out = data['test'][i]['output']
t_pred = data['test'][i]['predict']
ans = flattener(t_pred)
 df_sub['output'][df_sub['output_id']=="{}_{}".format(idx, i)] = ans
return df_sub
<feature_engineering><EOS> 
df_sub.to_csv("submission.csv", index=False )