Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<train_model>
import numpy as np
import pandas as pd
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np
from scipy import ndimage
import tensorflow as tf
from tensorflow import keras
from keras.layers import Input, LSTM, Embedding, Dense
from keras.models import Model
from tqdm import tqdm
import itertools
def width_fitness(predicted, expected_output):

 return np.abs(predicted.shape[0] - expected_output.shape[0])
 def height_fitness(predicted, expected_output):

 return np.abs(predicted.shape[1] - expected_output.shape[1])
 def activated_pixels_fitness(p, e):

 shape =(max(p.shape[0], e.shape[0]), max(p.shape[1], e.shape[1]))
diff = np.zeros(shape, dtype=int)
 diff[0:p.shape[0], 0:p.shape[1]] =(p > 0 ).astype(int)
 diff[0:e.shape[0], 0:e.shape[1]] -=(e > 0 ).astype(int)
 return(diff != 0 ).sum()
def colors_fitness(p, e):
p_colors = np.unique(p)
 e_colors = np.unique(e)
 nb_inter = len(np.intersect1d(p_colors, e_colors))
return(len(p_colors)- nb_inter)+(len(e_colors)- nb_inter)
 fitness_functions = [colors_fitness, activated_pixels_fitness, height_fitness, width_fitness] 
<compute_test_metric>
def load_data(path):
tasks = pd.Series()
for file_path in os.listdir(path):
task_file = path_join(path, file_path)
 with open(task_file, 'r')as f:
task = json.load(f)
 tasks[file_path[:-5]] = task
return tasks
def product_less(a, b):

 a = np.array(a)
 b = np.array(b)
 return(np.array(a)< np.array(b)).all()<compute_test_metric>
train_tasks = load_data('.. /input/abstraction-and-reasoning-challenge/training/')
 evaluation_tasks = load_data('.. /input/abstraction-and-reasoning-challenge/evaluation/')
 test_tasks = load_data('.. /input/abstraction-and-reasoning-challenge/test/')
 train_tasks.head()
def evaluate_fitness(program, task):

 score = np.zeros(( len(fitness_functions)))
 for sample in task:
i = np.array(sample['input'])
 o = np.array(sample['output'])
 for index, fitness_function in enumerate(fitness_functions):
images = evaluate(program, i)
 if images == []:
score[index] += 500
else:
score[index] = fitness_function(images[0], o)
 return tuple(score )<randomize_order>
class fund_unit:
def __init__(self,colour,position,ambient = -1):
self.colour = colour
self.position = position
l1 = [x[0] for x in position]
l2 = [x[1] for x in position]
a = min(l1)
 corners = get_corners(position)
 self.top_right = corners['top_right']
self.top_left = corners['top_left']
self.bottom_left = corners['bottom_left']
self.bottom_right = corners['bottom_right']
self.colour = colour
self.ambient = ambient
self.right = corners['right']
self.left = corners['left']
self.bottom = corners['bottom']
self.top = corners['top']
def build_candidates(allowed_nodes=[identity], best_candidates=[], nb_candidates=200):

 new_candidates = []
length_limit = 4
def random_node() :
return random.choice(allowed_nodes)
 while(len(new_candidates)< nb_candidates):
for i in range(5):
new_candidates += [[random_node() ]]
for best_program in best_candidates:
if len(best_program)< length_limit - 1:
new_candidates += [[random_node() ] + best_program]
if len(best_program)< length_limit - 1:
new_candidates += [best_program + [random_node() ]]
new_candidates += [list(best_program)]
new_candidates[-1][random.randrange(0, len(best_program)) ] = random_node()
np.random.shuffle(new_candidates)
 return new_candidates[:nb_candidates] 
<categorify>
def create(colour,position):
unit = fund_unit(colour,position)
 return(unit )
def remove_functions(all_functions, to_remove):
reduced_functions=[]
for function in all_functions:
fname=str(function.__name__)
 if '_lifted' in fname:
fname=fname[:-7]
if fname not in to_remove:
reduced_functions.append(function)
 return reduced_functions
def color_change_heuristics(all_functions_list, task):

 outputs=np.zeros(0)
 for test_or_train in [task['train'], task['test']]:
for x in test_or_train:
if len(x)==2:
outputs=np.append(outputs,x['output'])
 to_remove=[]
for color in range(0,10):
if color not in np.unique(outputs):
to_remove+=['to_c'+str(color)]
other_colors= list(range(0,10))
other_colors.remove(color)
 for other_color in other_colors:
to_remove+=['c'+str(other_color)+'_to_c'+str(color)]
inputs=np.zeros(0)
 for test_or_train in [task['train'], task['test']]:
for x in test_or_train:
inputs=np.append(inputs,x['input'])
 for color in range(1,10):
if color not in np.unique(inputs):
other_colors= list(range(1,10))
other_colors.remove(color)
 for other_color in other_colors:
to_remove+=['c'+str(color)+'_to_c'+str(other_color)]
return remove_functions(all_functions, to_remove)
 def initial_color_heuristic(all_functions_list, task):

 present_colors=np.zeros(0)
 for test_or_train in [task['train'], task['test']]:
for x in test_or_train:
present_colors=np.append(present_colors,x['input'])
 if len(x)==2:
present_colors=np.append(present_colors,x['output'])
 colors_not_present=[]
for colorcode in range(0,10):
if colorcode not in np.unique(present_colors):
colors_not_present=np.append(colors_not_present, colorcode)
 to_remove=[]
for colorcode in colors_not_present:
for function in all_functions_list:
if 'c'+str(int(colorcode)) in function.__name__:
fname=function.__name__
if '_lifted' in fname:
fname=fname[:-7]
to_remove+=[fname]
return remove_functions(all_functions, to_remove)
 def heuristics(all_functions_list, task):
reduced_functions=initial_color_heuristic(all_functions_list, task)
 reduced_functions=color_change_heuristics(reduced_functions, task)
 return reduced_functions<categorify>
def min_steps(unit1,unit2,include_end = 0):
l = []
for x in unit1.position:
for y in unit2.position:
a = count_steps(x[0],y[0],include_end)
 b = count_steps(x[1],y[1],include_end)
 l.append(( a,b))
l.sort(key = lambda x : abs(x[0])+ abs(x[1]))
t = l[0]
return(np.array([t[0],t[1]]))
def build_model(task, candidates_nodes=all_functions, max_iterations=20, verbose=True):
if verbose:
print("Candidates nodes are:", [program_desc([n])for n in candidates_nodes])
 print()
best_candidates = {}
for i in range(max_iterations):
if verbose:
print("Iteration ", i+1)
 print("-" * 10)
 candidates = build_candidates(candidates_nodes, best_candidates.values())
 for candidate in candidates:
score = evaluate_fitness(candidate, task)
 is_uncomparable = True
best_candidates_items = list(best_candidates.items())
 for best_score, best_candidate in best_candidates_items:
if product_less(score, best_score):
del best_candidates[best_score]
best_candidates[score] = candidate
is_uncomparable = False
if product_less(best_score, score)or best_score == score:
is_uncomparable = False
if is_uncomparable:
best_candidates[score] = candidate
for program in best_candidates.values() :
if is_solution(program, task):
return program
if verbose:
print("Best candidates lenght:", len(best_candidates))
random_candidate_score = random.choice(list(best_candidates.keys()))
print("Random candidate score:", random_candidate_score)
 print("Random candidate implementation:", program_desc(best_candidates[random_candidate_score]))
return None<import_modules>
def max_steps(unit1,unit2,include_end = 0):
l = []
for x in unit1.position:
for y in unit2.position:
a = count_steps(x[0],y[0],include_end)
 b = count_steps(x[1],y[1],include_end)
 l.append(( a,b))
l.sort(key = lambda x : abs(x[0])+ abs(x[1]))
t = l[-1]
return(np.array([t[0],t[1]]))
import numpy as np
import pandas as pd
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np<define_variables>
def count_steps(a,b,include_end = 0):
steps = 0
if include_end == 0:
if a < b:
steps = b-a-1
if a > b:
steps = b-a+1
else:
steps = b-a
return(steps )
input_path = "/kaggle/input/abstraction-and-reasoning-challenge"
for dirname, _, filenames in os.walk(input_path):
print(dirname )<define_variables>
def get_size_list(units):
units.sort(key = lambda u : len(u.position))
return(units )
data_path = Path(input_path)
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'<load_from_disk>
def pattern_matching(unit1,unit2,with_colour = False):
if with_colour and not unit1.colour == unit2.colour:
return([])
 else:
l = []
matches = []
y = unit2.position[0]
for x in unit1.position:
l.append(x-y)
 s1 = [(a[0],a[1])for a in unit1.position]
s1 = set(s1)
 for z in l:
p2 = [y + z for y in unit2.position]
p2 = [(a[0],a[1])for a in p2]
p2 = set(p2)
 if p2.issubset(s1):
p2 = list(p2)
 p2 = [np.array([a[0],a[1]])for a in p2]
matches.append([p2,z])
 matches.sort(key = lambda a : abs(a[1][0])+ abs(a[1][1]))
return(matches )
task_file = str(training_path / '00d62c1b.json')
 with open(task_file, 'r')as f:
task = json.load(f)
 print(task.keys() )<define_variables>
def sort_by_size(l):
l.sort(key = lambda u : len(u.position))
return(l )
n_train_pairs = len(task['train'])
 n_test_pairs = len(task['test'])
 print(f'task contains {n_train_pairs} training pairs')
 print(f'task contains {n_test_pairs} test pairs' )<load_from_csv>
def sort_by_position(l,base = 0):
if base != 3 and base !=7:
l.sort(key = lambda u :(give_base(u,base)[0],give_base(u,base)[1]))
if base == 7:
l.sort(key = lambda u :(give_base(u,base)[1],give_base(u,base)[0]))
if base == 3:
l.sort(key = lambda u :(-give_base(u,base)[1],give_base(u,base)[0]))
return(l)
 def give_base(unit,base):
d = {}
d[0] = unit.top_left
d[1] = unit.top
d[2] = unit.top_right
d[3] = unit.right
d[4] = unit.bottom_right
d[5] = unit.bottom
d[6] = unit.bottom_left
d[7] = unit.left
return(d[base] )
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )<categorify>
def change_colour(l,c):
for unit in l:
unit.colour = c
return(l )
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<concatenate>
def get_units(img,with_diag = False):
img = np.array(img)
 r,c = img.shape
ambient = img.shape
E = {}
V = []
for i in range(r):
for j in range(c):
V.append(( i,j))
E[(i,j)] = []
for i in range(r):
for j in range(c):
if i-1 >= 0 and img[i-1,j] == img[i,j]:
E[(i,j)].append(( i-1,j))
if i+1 <= r-1 and img[i+1,j] == img[i,j]:
E[(i,j)].append(( i+1,j))
if j-1 >= 0 and img[i,j-1] == img[i,j]:
E[(i,j)].append(( i,j-1))
if j+1 <= c-1 and img[i,j+1] == img[i,j]:
E[(i,j)].append(( i,j+1))
if with_diag:
if i-1 >= 0 and j-1 >= 0 and img[i-1,j-1] == img[i,j]:
E[(i,j)].append(( i-1,j-1))
if i-1 >= 0 and j+1 <= c-1 and img[i-1,j+1] == img[i,j]:
E[(i,j)].append(( i-1,j+1))
if i+1 <= r-1 and j-1 >= 0 and img[i+1,j-1] == img[i,j]:
E[(i,j)].append(( i+1,j-1))
if i+1 <= r-1 and j+1 <= c-1 and img[i+1,j+1] == img[i,j]:
E[(i,j)].append(( i+1,j+1))
parent = DFS(V,E)
 units = []
for v in parent.keys() :
if len(parent[v])> 0:
temp = [np.array([x[0],x[1]])for x in parent[v]]
unit = fund_unit(img[v],temp,ambient)
 units.append(unit)
 units.sort(key = lambda unit:(unit.top_left[0],unit.top_left[1]))
return(units )
example_grid = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
display(example_grid)
 print(flattener(example_grid))<load_from_disk>
def offset(coordinates,a,b):
new_x = coordinates[0] + a
new_y = coordinates[1] + b
return(np.array([new_x,new_y]))
def read_json_file(fileName):
with open(fileName, 'r')as f:
return json.load(f )<categorify>
def DFS(V,E):
colour = {}
parent = {}
for v in V:
colour[v] = 0
parent[v] = []
for v in V:
if colour[v] == 0:
DFS_search(v,E,parent[v],colour)
 return(parent )
def task_train001(df_input):
df_input = np.array(df_input)
 df_output = np.array(df_output)
 color_input, color_output = input_all_colors[0], new_colors[0]
def get_closed_area(arr):
H, W = arr.shape
Dy = [0, -1, 0, 1]
Dx = [1, 0, -1, 0]
arr_padded = np.pad(arr,(( 1, 1),(1, 1)) , "constant", constant_values=0)
 searched = np.zeros(arr_padded.shape, dtype=bool)
 searched[0, 0] = True
q = [(0, 0)]
while q:
y, x = q.pop()
for dy, dx in zip(Dy, Dx):
y_, x_ = y + dy, x + dx
if not 0 <= y_ < H + 2 or not 0 <= x_ < W + 2:
continue
if not searched[y_][x_] and arr_padded[y_][x_] == 0:
q.append(( y_, x_))
searched[y_, x_] = True
res = searched[1:-1, 1:-1]
res |= arr == color_input
return ~res
output = df_input.copy()
output[get_closed_area(df_input)] = color_output
return output
def wrapper_task_train001(df_train, df_test_item):
for df in df_train:
input_all_colors = [x for x in list(np.unique(df['input'])) if x != 0]
output_all_colors = [x for x in list(np.unique(df['output'])) if x != 0]
new_colors = [x for x in output_all_colors if x not in input_all_colors]
assert(len(new_colors)== 1)
 remove_border = False
if len(output_all_colors)- len(input_all_colors)== 0:
remove_border = True
def task_train001(x):
x = np.array(x)
 green, yellow = input_all_colors[0], new_colors[0]
def get_closed_area(arr):
H, W = arr.shape
Dy = [0, -1, 0, 1]
Dx = [1, 0, -1, 0]
arr_padded = np.pad(arr,(( 1, 1),(1, 1)) , "constant", constant_values=0)
 searched = np.zeros(arr_padded.shape, dtype=bool)
 searched[0, 0] = True
q = [(0, 0)]
while q:
y, x = q.pop()
for dy, dx in zip(Dy, Dx):
y_, x_ = y + dy, x + dx
if not 0 <= y_ < H + 2 or not 0 <= x_ < W + 2:
continue
if not searched[y_][x_] and arr_padded[y_][x_] == 0:
q.append(( y_, x_))
searched[y_, x_] = True
res = searched[1:-1, 1:-1]
res |= arr == green
return ~res
y = x.copy()
y[get_closed_area(x)] = yellow
if remove_border:
y[y == input_all_colors[0]] = 0
return y
for df in df_train:
assert(np.array_equal(task_train001(df['input']), df['output'])== True)
 print(np.array_equal(task_train001(df_test_item['input']), df_test_item['output']))
return task_train001(df_test_item['input'] )<load_from_csv>
def DFS_search(v,E,l,colour):
l.append(v)
 colour[v] = 1
for e in E[v]:
if colour[e] == 0:
DFS_search(e,E,l,colour)
 colour[v] = 2
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 used_k = len(submission)
 for output_id in tqdm_notebook(submission.index[:used_k], total = len(submission.index[:used_k])) :
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 fileName = str(test_path / str(task_id + '.json'))
task = read_json_file(fileName)
 try:
res = wrapper_task_train001(task['train'], task['test'][pair_id])
 pred_1 = flattener(res.tolist())
 print("wrapper_task_train001 success", pred_1)
 except:
data = task['test'][pair_id]['input']
pred_1 = flattener(data)
 pred = pred_1
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv' )<import_modules>
def do_for_list(l,f):
t = []
for x in l:
t = t + f(x)
 return(t )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
from sklearn.preprocessing import Imputer
from sklearn import preprocessing
from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score
import re<load_from_csv>
def check_contractible(unit,with_diag = False):
g = get_genus(unit,with_diag)
 if g == 0:
return(True)
 else:
return(False )
massey = pd.read_csv('.. /input/MasseyOrdinals.csv')
 cities = pd.read_csv('.. /input/Cities.csv')
 conferences = pd.read_csv('.. /input/Conferences.csv')
 conf_tourney = pd.read_csv('.. /input/ConferenceTourneyGames.csv')
 game_cities = pd.read_csv('.. /input/GameCities.csv')
 tourney_results = pd.read_csv('.. /input/NCAATourneyCompactResults.csv')
 detailed_tourney_results = pd.read_csv('.. /input/NCAATourneyDetailedResults.csv')
 tourney_all = pd.read_csv('.. /input/NCAATourneySeedRoundSlots.csv')
 tourney_seeds = pd.read_csv('.. /input/NCAATourneySeeds.csv')
 tourney_slots = pd.read_csv('.. /input/NCAATourneySlots.csv')
 reg_season = pd.read_csv('.. /input/RegularSeasonCompactResults.csv')
 detailed_reg_season = pd.read_csv('.. /input/RegularSeasonDetailedResults.csv')
 seasons = pd.read_csv('.. /input/Seasons.csv')
 second_tourney = pd.read_csv('.. /input/SecondaryTourneyCompactResults.csv')
 detailed_second_tourney = pd.read_csv('.. /input/SecondaryTourneyTeams.csv')
 team_coaches = pd.read_csv('.. /input/TeamCoaches.csv')
 team_conferences = pd.read_csv('.. /input/TeamConferences.csv')
 teams = pd.read_csv('.. /input/Teams.csv')
 teams_description = pd.read_csv('.. /input/Teams.csv' )<groupby>
def get_non_contractible(units,with_diag = False,with_background = False):
non_contract = []
units = sort_by_size(units)
 if not with_background and units[-1] == 0:
units = units[:-1]
for u in units:
if not check_contractible(u,with_diag):
non_contract.append(u)
 return(non_contract )
detailed_reg_season.groupby('Season' ).count()<load_from_csv>
def get_up_to_homotopy(units):
t = []
for u in units:
if not check_contractible(u):
t.append(u)
 return(t )
rs = pd.read_csv(".. /input/RegularSeasonCompactResults.csv")
 results_data = pd.read_csv(".. /input/RegularSeasonCompactResults.csv")
 K = 45
HOME_ADVANTAGE = 10
rs = results_data
rs.head(3)
 team_ids = set(rs.WTeamID ).union(set(rs.LTeamID))
len(team_ids)
 elo_dict = dict(zip(list(team_ids), [1500] * len(team_ids)))
 rs['margin'] = rs.WScore - rs.LScore
def elo_pred(elo1, elo2):
return(1./(10.**(-(elo1 - elo2)/ 400.) + 1.))
 def expected_margin(elo_diff):
return(( 7.5 + 0.006 * elo_diff))
def elo_update(w_elo, l_elo, margin):
elo_diff = w_elo - l_elo
pred = elo_pred(w_elo, l_elo)
 mult =(( margin + 3.) ** 0.8)/ expected_margin(elo_diff)
 update = K * mult *(1 - pred)
 return(pred, update)
 assert np.all(rs.index.values == np.array(range(rs.shape[0]))), "Index is out of order."
preds = []
w_elo = []
l_elo = []
for row in rs.itertuples() :
w = row.WTeamID
l = row.LTeamID
margin = row.margin
wloc = row.WLoc
w_ad, l_ad, = 0., 0.
 if wloc == "H":
w_ad += HOME_ADVANTAGE
elif wloc == "A":
l_ad += HOME_ADVANTAGE
pred, update = elo_update(elo_dict[w] + w_ad,
elo_dict[l] + l_ad,
margin)
 elo_dict[w] += update
elo_dict[l] -= update
preds.append(pred)
 w_elo.append(elo_dict[w])
 l_elo.append(elo_dict[l])
 rs['w_elo'] = w_elo
rs['l_elo'] = l_elo
rs['elo_dif'] =rs['w_elo']-rs['l_elo']
print(rs[['elo_dif','margin']].corr().iloc[0,1])
<create_dataframe>
def check_identical(l):
u0 = l[0]
t = 0
for u in l:
if len(u.position)== len(u0.position):
p = pattern_matching(u,u0)
 if len(p)>= 1:
t += 1
if t == len(l):
return(True)
 else:
return(False )
rs_win=rs[['Season','DayNum','WTeamID','w_elo']]
rs_lose=rs[['Season','DayNum','LTeamID','l_elo']]
rs_win=rs_win.rename(columns={'WTeamID':'TeamID','w_elo':'elo'})
 rs_lose=rs_lose.rename(columns={'LTeamID':'TeamID','l_elo':'elo'})
 rs_all=rs_win.append(rs_lose)
 final_elo=pd.DataFrame()
for team in rs_all['TeamID'].unique() :
for season in rs_all[rs_all['TeamID']==team]['Season'].unique() :
append=rs_all[(rs_all['TeamID']==team)&(rs_all['Season']==season)].sort_values('DayNum', ascending=False ).iloc[0,:]
final_elo=final_elo.append(append)
 final_elo.head(2 )<load_from_csv>
def get_background(img,get_diag = False):
img = np.array(img)
 ambient = np.array(img ).shape
units = get_units(img,get_diag)
 background = sort_by_size(units)[-1]
return(background )
tourney_seeds = pd.read_csv('.. /input/NCAATourneySeeds.csv' )<drop_column>
def check_actual_coord(x,y,r,c):
if x >= 0 and x <= r-1 and y >= 0 and y <= c-1:
return(True)
 else:
return(False )
def removeletters(x):
return re.sub('[^0-9]','', x )<feature_engineering>
def udlr(x,r,c):
a,b = x[0],x[1]
l = []
x_up = [a-1,b]
x_down = [a+1,b]
x_right = [a,b+1]
x_left = [a,b-1]
t = [x_up,x_down,x_right,x_left]
for i in range(4):
if check_actual_coord(t[i][0],t[i][1],r,c):
l.append(np.array(t[i]))
return(l )
tourney_seeds['Seed']=tourney_seeds['Seed'].apply(lambda x: removeletters(x)).apply(int )<merge>
def is_surrounded(u1,u2):
r,c = u1.ambient
val = True
for x in u1.position:
t = udlr(x,r,c)
 m = len(t)
 if m < 4:
val = False
for i in range(m):
if not check_belongs_to([t[i]],u1.position):
if not check_belongs_to([t[i]],u2.position):
val = False
return(val )
df1=tourney_seeds.merge(tourney_results, left_on= ['TeamID','Season'], right_on=['WTeamID','Season'], how='left' )<feature_engineering>
t = np.array([[1,1,1,1],[1,1,0,1],[1,0,0,1],[1,1,1,1]])
 units = sort_by_size(get_units(t))
df1['TeamID1']=df1['WTeamID']
df1['TeamID2']=df1['LTeamID']
df1['Win']=np.where(df1['TeamID1']==df1['WTeamID'],1,0)
 df1=df1.rename(columns={'Seed':'TeamID1Seed'} )<merge>
def get_surrounded(units):
surrounded = {}
for u in units:
surrounded[u] = []
for v in units:
if is_surrounded(v,u):
surrounded[u].append(v)
 return(surrounded )
df2=df1.merge(tourney_seeds, left_on= ['LTeamID','Season'], right_on=['TeamID','Season'], how='right')
 df2=df2.rename(columns={'Seed':'TeamID2Seed','Win':'Team1Win'})
 df2.drop(['TeamID_x','TeamID_y','NumOT','WLoc','LTeamID','WTeamID'], 1, inplace = True )<drop_column>
def surrounded_list(units):
surrounded = []
d = get_surrounded(units)
 l = []
for k in d.keys() :
l = l + d[k]
return(l )
df2['Margin']=df2['WScore']-df2['LScore']
df2.drop(['WScore','LScore'], 1, inplace = True )<feature_engineering>
def boundary_img(r,c):
l = []
for j in range(c):
l.append(np.array([0,j]))
for j in range(r):
l.append(np.array([j,c-1]))
for j in range(c):
l.append(np.array([r-1,j]))
for j in range(r):
l.append(np.array([j,0]))
return(l )
lose_df=df2[['TeamID2','TeamID1','TeamID2Seed','TeamID1Seed','Season']]
lose_df=lose_df.rename(columns={'TeamID2':'TeamID','TeamID1':'OppTeamID','TeamID2Seed':'Seed','TeamID1Seed':'OppSeed'})
 lose_df['Win']=0<define_variables>
def get_intersection(l1,l2):
l11 = [(x[0],x[1])for x in l1]
l22 = [(x[0],x[1])for x in l2]
l = list(set(l11)&set(l22))
l = [np.array(x)for x in l]
return(l )
win_df=df2[['TeamID1','TeamID2','TeamID1Seed','TeamID2Seed','Season']]
win_df['Win']=1
win_df=win_df.rename(columns={'TeamID1':'TeamID','TeamID2':'OppTeamID','TeamID1Seed':'Seed','TeamID2Seed':'OppSeed'} )<concatenate>
def check_empty_intersection(l1,l2):
l = get_intersection(l1,l2)
 if len(l)> 0:
return(False)
 else:
return(True )
all_tourney_df=win_df.append(lose_df )<merge>
def get_enclosed_area(unit):
r,c = unit.ambient
l = boundary_img(r,c)
 i = 0
count = 0
while check_belongs_to([l[i]],unit.position):
count += 1
i += 1
if i > len(l)-1:
break
if count < 2*c + 2*r - 4:
start_key = l[i]
is_a_boundary = False
else:
is_a_boundary = True
u = fund_unit(1,unit.position,unit.ambient)
 img = concat([u],u.ambient)
 units = get_units(img)
 enclosed_area = []
if is_a_boundary:
for v in units:
if check_belongs_to([v.top_left],[u.top_left]):
enclosed_area += v.position
if len(units)> 2 and not is_a_boundary:
for v in units:
if not check_belongs_to([v.top_left],[u.top_left])and check_empty_intersection(l,v.position):
enclosed_area += v.position
return(enclosed_area )
all_tourney_df=all_tourney_df.merge(final_elo, left_on=['OppTeamID','Season'], right_on=['TeamID','Season'],how='left')
 all_tourney_df.drop(['TeamID_y'], 1, inplace = True)
 all_tourney_df=all_tourney_df.rename(columns={'TeamID_x':'TeamID','elo':'Oppelo'} )<merge>
def contained_list(units):
contained = []
enclosed_area = []
for u in units:
if u.colour != 0:
enclosed_area += get_enclosed_area(u)
 for v in units:
if check_belongs_to(v.position,enclosed_area):
contained.append(v)
 return(contained )
all_tourney_df=all_tourney_df.merge(final_elo, left_on=['TeamID','Season'], right_on=['TeamID','Season'],how='left' )<drop_column>
def contained_in(units):
d = {}
l = detect_regular_non_contractible(units)
 for u in l:
d[u] = []
x = u.top_left
y = u.bottom_right
l1 = list(range(x[0],y[0]+1))
l2 = list(range(x[1],y[1]+1))
temp = list(itertools.product(l1,l2))
p = []
for t in temp:
p.append(np.array([t[0],t[1]]))
pdiff = []
for v in p:
r0 = 0
for z in u.position:
if v[0] == z[0] and v[1] == z[1]:
r0 += 1
if r0 == 0:
pdiff.append(v)
 udiff = fund_unit(0,pdiff,u.ambient)
 for v in units:
if check_belongs_to(v.position,u.position):
d[u].append(v)
 return(d )
all_tourney_df.drop(['DayNum_y','DayNum_x'], 1, inplace = True )<filter>
def get_regular_shapes(units):
d = {}
d['square'] = []
d['vertical line'] = []
d['horizontal line'] = []
d['horizontal rectangle'] = []
d['vertical rectangle'] = []
d['-1'] = []
for u in units:
d[is_regular(u)].append(u)
 return(d )
all_tourney_df=all_tourney_df[(~all_tourney_df['TeamID'].isnull())&(~all_tourney_df['OppSeed'].isnull())]<feature_engineering>
def shape_equivalences(units,return_lists = True,remove_background = False):
units = sort_by_size(units)
 if remove_background:
units = units[:-1]
d = {}
d[units[0]] = []
is_key = {}
for u in units:
is_key[u] = False
for u in units:
c = 0
for v in units:
if is_key[v]:
if len(u.position)== len(v.position):
l = pattern_matching(u,v)
 if len(l)> 0:
c += 1
d[v].append(u)
 if c == 0:
d[u] = [u]
is_key[u] = True
m = len(d.keys())
 shape_equivalences = {}
for j,key in enumerate(d.keys()):
for u in d[key]:
shape_equivalences[u] = j
if return_lists:
return(d)
 else:
return(shape_equivalences )
all_tourney_df['elodif']=all_tourney_df['elo']-all_tourney_df['Oppelo']
all_tourney_df['seeddif']=all_tourney_df['Seed']-all_tourney_df['OppSeed']<prepare_x_and_y>
def get_task_num(img,task):
m = len(task['train'])
 for i in range(m):
if np.all(img == task['train'][i]['input']):
return(i )
features=all_tourney_df[['TeamID','OppSeed','Seed','Season','OppTeamID','elodif']]
target=all_tourney_df['Win']<compute_train_metric>
def common_shapes_across_images(img,task,img_units,num):
m = len(task['train'])
 all_units = []
task_number = {}
for i in range(m):
if i != num:
units = get_units(task['train'][i]['input'])
 else:
units = img_units
for u in units:
task_number[u] = i
all_units += units
if num == -1:
units = img_units
all_units += img_units
for u in units:
task_number[u] = i
d = shape_equivalences(all_units)
 shapes_across_images = []
for x in d.keys() :
l = [task_number[y] for y in d[x]]
if len(set(l)) == m:
shapes_across_images.append(x)
 final_list = []
for x in shapes_across_images:
for u in d[x]:
if task_number[u] == num:
final_list.append(u)
 return(final_list )
features_train, features_test, target_train, target_test = train_test_split(features,
target, test_size = 0.3)
 target_test
clf = GaussianNB()
clf.fit(features_train, target_train)
 target_pred = clf.predict(features_test)
 accuracy_score(target_test, target_pred, normalize = True)
 def conf_matrix(y_test, y_predict):
conf = sk_confusion_matrix(y_test, y_predict)
 conf = pd.DataFrame(data=conf)
 conf.columns.name = 'Predicted label'
conf.index.name = 'Actual label'
return conf
conf_matrix(target_test, target_pred )<import_modules>
def shape_equivalences_counts(units,remove_background = False):
d = {}
s = shape_equivalences(units,True,remove_background)
 for x in s.keys() :
m = len(s[x])
 if m in d.keys() :
d[m] += s[x]
else:
d[m] = s[x]
return(d )
from sklearn import linear_model, datasets<compute_train_metric>
def check_belongs_to(l1,l2):
r = 0
for z in l1:
for x in l2:
if x[0] == z[0] and x[1] == z[1]:
r += 1
if r == len(l1):
return(True)
 else:
return(False )
clf = linear_model.LogisticRegression()
clf.fit(features_train, target_train)
 target_pred = clf.predict(features_test)
 accuracy_score(target_test, target_pred, normalize = True )<compute_test_metric>
def check_this_please(u,x1,x2):
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]]))
if check_belongs_to(l1,u.position):
count = 1
else:
count = 0
return(count )
print(metrics.classification_report(target_test, target_pred))<filter>
def detect_regular_non_contractible(units):
units = sort_by_size(units)
 regular_non_contractible = []
for u in units:
x = u.top_left
y = u.bottom_right
l1 = list(range(x[0],y[0]+1))
l2 = list(range(x[1],y[1]+1))
temp = list(itertools.product(l1,l2))
p = []
for t in temp:
p.append(np.array([t[0],t[1]]))
pdiff = []
for v in p:
r0 = 0
for z in u.position:
if v[0] == z[0] and v[1] == z[1]:
r0 += 1
if r0 == 0:
pdiff.append(v)
 if len(pdiff)> 0:
udiff = fund_unit(0,pdiff,u.ambient)
 u0 = fund_unit(0,p,u.ambient)
 s = is_regular(udiff)
 if s != '-1':
count = 0
x1 = udiff.top_left
x2 = udiff.top_right
l1 = list(range(x1[1],x2[1]+1))
l2 = []
for t in l1:
l2.append(np.array([x1[0]-1,t]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.top_right
x2 = udiff.bottom_right
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]+1]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.bottom_left
x2 = udiff.bottom_right
l1 = list(range(x1[1],x2[1]+1))
l2 = []
for t in l1:
l2.append(np.array([x1[0]+1,t]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.top_left
x2 = udiff.bottom_left
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]-1]))
if check_belongs_to(l2,u.position):
count += 1
if count == 4:
regular_non_contractible.append(u)
 return(regular_non_contractible )
df2018=tourney_seeds[tourney_seeds['Season']==2018]<create_dataframe>
def filter_by_size(units):
f = {}
for j in range(1,7):
f[j] = []
for u in units:
if len(u.position)< 6:
f[len(u.position)].append(u)
 else:
f[6].append(u)
 return(f )
matchups=pd.DataFrame()
index=0
for team in teams2018:
for team2 in teams2018:
if team!=team2:
index=index+1
matchup=pd.DataFrame({'TeamID':team,'OppTeamID':team2},index=[index])
 matchups=matchups.append(matchup )<filter>
def concat(l,ambient,background = 0):
r,c = ambient
a = np.full(ambient,background)
 for x in l:
for z in x.position:
if z[0] >= 0 and z[0] <= r-1 and z[1] >= 0 and z[1] <= c-1:
a[z[0],z[1]] = x.colour
return(a )
final_elo2018=final_elo[final_elo['Season']==2018][['TeamID','elo']]<merge>
def input_output_shape_is_same(task):
return all([np.array(el['input'] ).shape == np.array(el['output'] ).shape for el in task['train']])
 def get_equals(task_series):
l = []
for j in task_series.index:
if input_output_shape_is_same(task_series[j]):
l.append(j)
 return(l )
matchups=matchups.merge(final_elo2018, left_on=['OppTeamID'],right_on=['TeamID'])
 matchups.drop(labels=['TeamID_y'], inplace=True, axis=1)
 matchups=matchups.rename(columns={'TeamID_x':'TeamID'})
 matchups=matchups.rename(columns={'elo':'Oppelo'} )<merge>
l_input_output_same_size = get_equals(train_tasks)
 l_input_output_same_size.sort()
matchups=matchups.merge(final_elo2018, left_on=['TeamID'],right_on=['TeamID'])
 matchups=matchups.merge(df2018, left_on=['OppTeamID'], right_on=['TeamID'])
 matchups=matchups.rename(columns={'TeamID_x':'TeamID','Seed':'OppSeed'})
 matchups.drop(['TeamID_y'], 1, inplace = True )<merge>
def convert_to_tuple(t):
s = [(a[0],a[1])for a in t]
return(s )
matchups=matchups.merge(df2018, left_on=['TeamID'], right_on=['TeamID'])
 matchups=matchups.rename(columns={'Season_x':'Season'})
 matchups.drop(['Season_y'], 1, inplace = True)
 matchups['elodif']=matchups['elo']-matchups['Oppelo']
matchups['seeddif']=matchups['Seed']-matchups['OppSeed']<define_variables>
l_colour_match_no_diag = []
for index in l_input_output_same_size:
task = train_tasks[index]
c = 0
for example in task['train']:
units_input = get_units(np.array(example['input']))
units_output = get_units(np.array(example['output']))
if len(units_input)== len(units_output):
sub_count = 0
for u in units_input:
s1 = set(convert_to_tuple(u.position))
for v in units_output:
s2 = set(convert_to_tuple(v.position))
if s1 == s2:
sub_count += 1
if sub_count == len(units_input):
c += 1
if c == len(task['train']):
l_colour_match_no_diag.append(index )
features=matchups[['TeamID','OppSeed','Seed','Season','OppTeamID','elodif']]<predict_on_test>
def get_lists(img,task = None,num = -1,with_diag = False):
list_dict = {}
img = np.array(img)
 ambient = np.array(img ).shape
units = get_units(img,with_diag)
 list_dict[0] = units
list_dict[1] = get_non_contractible(units,with_diag,False)
 list_dict[2] = surrounded_list(units)
 list_dict[3] = contained_list(units)
 shape_d = get_regular_shapes(units)
 list_dict[4] = shape_d['square']
list_dict[5] = shape_d['vertical line']
list_dict[6] = shape_d['horizontal line']
list_dict[7] = shape_d['horizontal rectangle']
list_dict[8] = shape_d['vertical rectangle']
list_dict[9] = shape_d['-1']
list_dict[10] = list_dict[4] + list_dict[5] + list_dict[6] + list_dict[7] + list_dict[8]
list_dict[11] = detect_regular_non_contractible(units)
 shape_counts = shape_equivalences_counts(units,True)
 for i in range(1,4):
if i not in shape_counts.keys() :
shape_counts[i] = []
for i in range(1,4):
list_dict[11+i] = shape_counts[i]
list_dict[15] = common_shapes_across_images(img,task,units,num)
 f = filter_by_size(units)
 for j in range(1,7):
list_dict[15+j] = f[j]
list_dict = get_list_dict_complements(units,list_dict)
 return(list_dict )
probabilities=clf.predict_proba(features )<define_variables>
def get_list_dict_complements(units,list_dict):
e = {}
for i in list_dict.keys() :
if i > 0:
t = list(set(units)- set(list_dict[i]))
e[-i] = t
for j in e.keys() :
list_dict[j] = e[j]
return(list_dict )
predictions=[]
for i in probabilities:
predictions.append(( i[1]))<prepare_output>
def extract_lists_from_dict(d):
t = []
for x in d.keys() :
t.append(d[x])
 return(t )
matchups['prediction']=predictions<find_best_params>
def move_in_evens(l):
m = len(l)
 if m > 0:
r = []
c = 0
while c < m:
r.append(c)
 c += 2
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )
teams_description=teams_description.iloc[:,:2]<merge>
def move_in_odds(l):
m = len(l)
 if m > 0:
r = []
c = 1
while c < m:
r.append(c)
 c += 2
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )
output=matchups.merge(teams_description, left_on=['OppTeamID'], right_on=['TeamID'])
 output=output.rename(columns={'TeamName':'OppTeamName'})
 output.drop(labels=['TeamID_y'], inplace=True, axis=1 )<merge>
def move_in_3s(l):
m = len(l)
 if m > 0:
r = []
c = 0
while c < m:
r.append(c)
 c += 3
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )
output=output.merge(teams_description, left_on=['TeamID_x'], right_on=['TeamID'] )<rename_columns>
def middle(l):
m = len(l)
 if m > 0:
if m%2 == 0:
return([l[m//2]])
 if m%2 == 1:
t =(m-1)//2
return([l[t]])
 else:
return([] )
output=output.rename(columns={'TeamID_x':'TeamID'} )<drop_column>
def start_mid_end(l):
if len(l)> 0:
mid = middle(l)[0]
return([l[0],mid,l[-1]])
 else:
return([] )
output=output[['TeamName','OppTeamName','prediction','Seed','OppSeed','elo','Oppelo']]<save_to_csv>
def template_sublist(l,template):
d = {}
m = len(l)
 for i in range(4):
if m > i:
d[i] = [l[i]]
else:
d[i] = []
for i in range(1,5):
if m >= i:
d[3+i] = [l[-i]]
else:
d[3+i] = []
d[8]= l[1:]
d[9]= l[2:]
d[10]= l[3:]
d[11]= l[:-1]
d[12]= l[:-2]
d[13]= l[:-3]
d[14] = move_in_evens(l)
 d[15] = move_in_odds(l)
 d[16] = move_in_3s(l)
 d[17] = middle(l)
 d[18] = start_mid_end(l)
 d[19] = l
return(d[template] )
output.to_csv('descriptive_output.csv' )<merge>
def sorting_list(l,sorting_type):
if sorting_type <= 7:
return(sort_by_position(l,sorting_type))
if sorting_type == 8:
return(sort_by_size(l))
matchups.merge(teams_description, left_on=['TeamID'], right_on=['TeamID'] ).groupby('TeamName' ).mean().sort_values('prediction', ascending=False )<filter>
def get_list_selection(l,sorting_type,template,colour_filter):
l = sorting_list(l,sorting_type)
 l = template_sublist(l,template)
 c_dict = get_colour_dict(l)
 if colour_filter < 10:
t = c_dict[colour_filter]
else:
t = l
return(t )
matchups_for_submission=matchups[matchups['TeamID']<matchups['OppTeamID']]<feature_engineering>
def is_equal(l1,l2):
count1 = 0
count2 = 0
m1 = len(l1)
 m2 = len(l2)
 if m1 == 0 and m2 > 0:
return(False)
 if m1 > 0 and m2 == 0:
return(False)
 for u in l1:
for v in l2:
x = [(a[0],a[1])for a in u.position]
y = [(b[0],b[1])for b in v.position]
if set(x)== set(y)and u.colour == v.colour:
count1 +=1
for u in l2:
for v in l1:
x = [(a[0],a[1])for a in u.position]
y = [(b[0],b[1])for b in v.position]
if set(x)== set(y)and u.colour == v.colour:
count2 +=1
if count1 == len(l1)and count2 == len(l2):
return(True)
 else:
return(False )
matchups_for_submission['ID'] = '2018_' + matchups_for_submission.TeamID.map(str)+ "_" + matchups_for_submission.OppTeamID.map(str)
 matchups_for_submission['pred']=matchups_for_submission['prediction']<load_from_csv>
def get_colour_change_dict(task,task_number):
img_in = task['train'][task_number]['input']
img_out = task['train'][task_number]['output']
img_in = np.array(img_in)
 ambient = img_in.shape
img_out = np.array(img_out)
 units_in = sort_by_position(get_units(img_in),0)
 units_out = sort_by_position(get_units(img_out),0)
 m = len(units_in)
 colour_diffs = {}
for i in range(m):
if units_in[i].colour != units_out[i].colour:
if units_out[i].colour not in colour_diffs.keys() :
colour_diffs[units_out[i].colour] = [units_in[i]]
else:
colour_diffs[units_out[i].colour].append(units_in[i])
 return(colour_diffs )
sample = pd.read_csv('.. /input/SampleSubmissionStage2.csv')
 del sample['Pred']<categorify>
def intersection(lst1, lst2):
return list(set(lst1)& set(lst2))
matchups_for_submission=matchups_for_submission[['ID','pred']]<save_to_csv>
def whittle_candidates_down(task,task_number,colour_diff_solutions,with_diag = False):
img = np.array(task['train'][task_number]['input'])
 list_dict = get_lists(img,task,task_number,with_diag)
 colour_diff = get_colour_change_dict(task,task_number)
 for c in colour_diff.keys() :
if c in colour_diff_solutions.keys() :
temp = colour_diff_solutions[c]
else:
temp = []
p = len(temp)
 if p == 0:
img = np.array(task['train'][task_number]['input'])
 colour_diff_solutions[c] = identify_generic_list(colour_diff[c],img,task,task_number,with_diag)
 else:
temp1 = []
for i in range(p):
a,b,c1,d = temp[i]
t = get_list_selection(list_dict[a],b,c1,d)
 if is_equal(t,colour_diff[c]):
temp1.append(temp[i])
 colour_diff_solutions[c] = temp1
e = get_unique_dict_values(colour_diff_solutions)
 return(e )
matchups_for_submission.to_csv('submission.csv', index=None )<set_options>
def training_algo(task,with_diag = False):
m = len(task['train'])
 colour_diff = get_colour_change_dict(task,0)
 img = np.array(task['train'][0]['input'])
 colour_diff_solutions = {}
for c in colour_diff.keys() :
l_c = colour_diff[c]
solutions = identify_generic_list(l_c,img,task,0,with_diag)
 colour_diff_solutions[c] = solutions
for i in range(1,m):
colour_diff_solutions = whittle_candidates_down(task,i,colour_diff_solutions,with_diag)
 for i in range(m-1,-1,-1):
colour_diff_solutions = whittle_candidates_down(task,i,colour_diff_solutions,with_diag)
 return(colour_diff_solutions )
InteractiveShell.ast_node_interactivity = "all"
%matplotlib inline<train_model>
def collect_predictions(task,with_diag = False):
colour_diff_solutions = training_algo(task,with_diag)
 preds = []
m = len(task['test'])
 for i in range(m):
img_out = assemble_test(task,i,colour_diff_solutions,with_diag)
 preds.append(img_out)
 return(preds )
image_folder_path = "/kaggle/input/global-wheat-detection/train/"
chosen_image = cv2.imread(os.path.join(image_folder_path, "1ee6b9669.jpg")) [:,:,::-1]
plt.figure(figsize =(20,10))
plt.imshow(chosen_image )<import_modules>
def assemble_test(task,task_number,colour_diff_solutions,with_diag = False):
img = np.array(task['test'][task_number]['input'])
 ambient = img.shape
list_dict = get_lists(img,task,-1,with_diag)
 output_units = []
for c in colour_diff_solutions.keys() :
if len(colour_diff_solutions[c])> 0:
temp = filter_solutions(list_dict,colour_diff_solutions,c)
 else:
temp = []
for t in temp:
t.colour = c
output_units += temp
units = get_list_selection(list_dict[0],0,19,10)
 other_units = []
m = len(output_units)
 if m == 0:
other_units = units
else:
for u in units:
c = 0
for v in output_units:
if not is_equal([u],[v]):
c += 1
if c == m:
other_units.append(u)
 output_units = other_units + output_units
img_out = concat(output_units,ambient)
 return(img_out )
class InsectAugmentation(ImageOnlyTransform):

 def __init__(self, insects=2, dark_insect=False, always_apply=False, p=0.5):
super().__init__(always_apply, p)
 self.insects = insects
self.dark_insect = dark_insect
self.insects_folder = "/kaggle/input/bee-augmentation/"
def apply(self, image, **kwargs):

 n_insects = random.randint(1, self.insects)
 if not n_insects:
return image
height, width, _ = image.shape
insects_images = [im for im in os.listdir(self.insects_folder)if 'png' in im]
for _ in range(n_insects):
insect = cv2.cvtColor(cv2.imread(os.path.join(self.insects_folder, random.choice(insects_images))), cv2.COLOR_BGR2RGB)
 insect = cv2.flip(insect, random.choice([-1, 0, 1]))
insect = cv2.rotate(insect, random.choice([0, 1, 2]))
h_height, h_width, _ = insect.shape
roi_ho = random.randint(0, image.shape[0] - insect.shape[0])
 roi_wo = random.randint(0, image.shape[1] - insect.shape[1])
 roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(insect, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 if self.dark_insect:
img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 insect_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)
 else:
insect_fg = cv2.bitwise_and(insect, insect, mask=mask)
 dst = cv2.add(img_bg, insect_fg, dtype=cv2.CV_64F)
 image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return image<set_options>
def filter_solutions(list_dict,colour_diff_solutions,c):
e = {}
for x in colour_diff_solutions[c]:
s = get_list_selection(list_dict[x[0]],x[1],x[2],x[3])
 s = sort_by_position(s,0)
 s = tuple(s)
 if s in e.keys() :
e[s] += 1
else:
e[s] = 1
l = list(e.keys())
 l.sort(key = lambda x : e[x])
 t = list(l[-1])
 return(t )
img = InsectAugmentation(insects=2, always_apply=True )(image=chosen_image)['image']
plt.figure(figsize =(20,10))
plt.imshow(img )<load_pretrained>
task = train_tasks[l_colour_match_no_diag[31]]
img = task['train'][0]['input']
chosen_image = cv2.imread(os.path.join(image_folder_path, "1ee6b9669.jpg")) [:,:,::-1]
img = InsectAugmentation(insects=2, dark_insect=True, always_apply=True )(image=chosen_image)['image']
plt.figure(figsize =(20,10))
plt.imshow(img )<install_modules>
def check_colour_match(task):
c = 0
for example in task['train']:
units_input = get_units(np.array(example['input']))
units_output = get_units(np.array(example['output']))
if len(units_input)== len(units_output):
sub_count = 0
for u in units_input:
s1 = set(convert_to_tuple(u.position))
for v in units_output:
s2 = set(convert_to_tuple(v.position))
if s1 == s2:
sub_count += 1
if sub_count == len(units_input):
c += 1
if c == len(task['train']):
return(True)
 else:
return(False )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def make_prediction(tasks):
result = pd.Series()
for idx, task in tqdm(tasks.iteritems()):
if input_output_shape_is_same(task)and check_colour_match(task):
pred = collect_predictions(task)
 else:
pred = [el['input'] for el in task['test']]
for i, p in enumerate(pred):
result[f'{idx}_{i}'] = flattener(np.array(p ).tolist())
 return result
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
submission = make_prediction(test_tasks)
 submission.head()
<load_pretrained><EOS> 
submission = submission.reset_index()
submission.columns = ['output_id', 'output']
submission.to_csv('submission.csv', index=False)
 submission