Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<sort_values>
import numpy as np
import pandas as pd
def sort_by_size(l):
l.sort(key = lambda u : len(u.position))
return(l )<sort_values>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 data = pd.concat([train, test], sort=False )
def sort_by_position(l,base = 0):
if base != 3 and base !=7:
l.sort(key = lambda u :(give_base(u,base)[0],give_base(u,base)[1]))
if base == 7:
l.sort(key = lambda u :(give_base(u,base)[1],give_base(u,base)[0]))
if base == 3:
l.sort(key = lambda u :(-give_base(u,base)[1],give_base(u,base)[0]))
return(l)
 def give_base(unit,base):
d = {}
d[0] = unit.top_left
d[1] = unit.top
d[2] = unit.top_right
d[3] = unit.right
d[4] = unit.bottom_right
d[5] = unit.bottom
d[6] = unit.bottom_left
d[7] = unit.left
return(d[base] )<set_options>
data.isnull().sum()
def change_colour(l,c):
for unit in l:
unit.colour = c
return(l )<categorify>
data.isnull().sum()
def get_units(img,with_diag = False):
img = np.array(img)
 r,c = img.shape
ambient = img.shape
E = {}
V = []
for i in range(r):
for j in range(c):
V.append(( i,j))
E[(i,j)] = []
for i in range(r):
for j in range(c):
if i-1 >= 0 and img[i-1,j] == img[i,j]:
E[(i,j)].append(( i-1,j))
if i+1 <= r-1 and img[i+1,j] == img[i,j]:
E[(i,j)].append(( i+1,j))
if j-1 >= 0 and img[i,j-1] == img[i,j]:
E[(i,j)].append(( i,j-1))
if j+1 <= c-1 and img[i,j+1] == img[i,j]:
E[(i,j)].append(( i,j+1))
if with_diag:
if i-1 >= 0 and j-1 >= 0 and img[i-1,j-1] == img[i,j]:
E[(i,j)].append(( i-1,j-1))
if i-1 >= 0 and j+1 <= c-1 and img[i-1,j+1] == img[i,j]:
E[(i,j)].append(( i-1,j+1))
if i+1 <= r-1 and j-1 >= 0 and img[i+1,j-1] == img[i,j]:
E[(i,j)].append(( i+1,j-1))
if i+1 <= r-1 and j+1 <= c-1 and img[i+1,j+1] == img[i,j]:
E[(i,j)].append(( i+1,j+1))
parent = DFS(V,E)
 units = []
for v in parent.keys() :
if len(parent[v])> 0:
temp = [np.array([x[0],x[1]])for x in parent[v]]
unit = fund_unit(img[v],temp,ambient)
 units.append(unit)
 units.sort(key = lambda unit:(unit.top_left[0],unit.top_left[1]))
return(units )<train_model>
train.isnull().sum()
def offset(coordinates,a,b):
new_x = coordinates[0] + a
new_y = coordinates[1] + b
return(np.array([new_x,new_y]))<compute_test_metric>
train.isnull().sum()
def DFS(V,E):
colour = {}
parent = {}
for v in V:
colour[v] = 0
parent[v] = []
for v in V:
if colour[v] == 0:
DFS_search(v,E,parent[v],colour)
 return(parent )<compute_test_metric>
test.isnull().sum()
def DFS_search(v,E,l,colour):
l.append(v)
 colour[v] = 1
for e in E[v]:
if colour[e] == 0:
DFS_search(e,E,l,colour)
 colour[v] = 2<categorify>
test.isnull().sum()
def do_for_list(l,f):
t = []
for x in l:
t = t + f(x)
 return(t )<statistical_test>
data['Sex'].replace(['male','female'], [0, 1], inplace=True )
def check_contractible(unit,with_diag = False):
g = get_genus(unit,with_diag)
 if g == 0:
return(True)
 else:
return(False )<groupby>
data = data.loc[data['Embarked'].isnull() == False, :].copy()
print(data.isnull().sum())
def get_non_contractible(units,with_diag = False,with_background = False):
non_contract = []
units = sort_by_size(units)
 if not with_background and units[-1] == 0:
units = units[:-1]
for u in units:
if not check_contractible(u,with_diag):
non_contract.append(u)
 return(non_contract )<categorify>
data = pd.concat([data, data['Embarked'].str.get_dummies() ], axis=1)
 data.rename(columns={'S': 'Embarked_S', 'C': 'Embarked_C', 'Q': 'Embarked_Q'}, inplace=True)
 print(data.isnull().sum())
def get_up_to_homotopy(units):
t = []
for u in units:
if not check_contractible(u):
t.append(u)
 return(t )<compute_test_metric>
if data['Fare'].isnull().sum() != 0:
dict_lookup_fare = data.loc[:, ['Fare', 'Pclass', 'Embarked']].groupby(['Pclass', 'Embarked'] ).mean().to_dict().get('Fare')
 data.loc[data['Fare'].isnull() , 'Fare'] = [dict_lookup_fare.get(i)for i in list(zip(data.loc[data['Fare'].isnull() , 'Pclass'],
data.loc[data['Fare'].isnull() , 'Embarked'])) ]
print(data.isnull().sum() )
def check_identical(l):
u0 = l[0]
t = 0
for u in l:
if len(u.position)== len(u0.position):
p = pattern_matching(u,u0)
 if len(p)>= 1:
t += 1
if t == len(l):
return(True)
 else:
return(False )<categorify>
dict_mean_age_by_pclass = data.loc[:, ['Age', 'Pclass']].groupby('Pclass' ).mean().to_dict().get('Age')
 data.loc[data['Age'].isnull() , 'Age'] = data.loc[data['Age'].isnull() , 'Pclass'].apply(lambda x: round(dict_mean_age_by_pclass.get(x)) )
def get_background(img,get_diag = False):
img = np.array(img)
 ambient = np.array(img ).shape
units = get_units(img,get_diag)
 background = sort_by_size(units)[-1]
return(background )<compute_test_metric>
print(data.isnull().sum())
def check_actual_coord(x,y,r,c):
if x >= 0 and x <= r-1 and y >= 0 and y <= c-1:
return(True)
 else:
return(False )<concatenate>
data['FamilySize'] = data['Parch'] + data['SibSp'] + 1
def udlr(x,r,c):
a,b = x[0],x[1]
l = []
x_up = [a-1,b]
x_down = [a+1,b]
x_right = [a,b+1]
x_left = [a,b-1]
t = [x_up,x_down,x_right,x_left]
for i in range(4):
if check_actual_coord(t[i][0],t[i][1],r,c):
l.append(np.array(t[i]))
return(l )<normalization>
data['IsAlone'] = 0
data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1
def is_surrounded(u1,u2):
r,c = u1.ambient
val = True
for x in u1.position:
t = udlr(x,r,c)
 m = len(t)
 if m < 4:
val = False
for i in range(m):
if not check_belongs_to([t[i]],u1.position):
if not check_belongs_to([t[i]],u2.position):
val = False
return(val )<define_search_space>
data.isnull().sum()
t = np.array([[1,1,1,1],[1,1,0,1],[1,0,0,1],[1,1,1,1]])
 units = sort_by_size(get_units(t))<categorify>
data.isnull().sum()
def get_surrounded(units):
surrounded = {}
for u in units:
surrounded[u] = []
for v in units:
if is_surrounded(v,u):
surrounded[u].append(v)
 return(surrounded )<categorify>
data['Title'] = data['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
def surrounded_list(units):
surrounded = []
d = get_surrounded(units)
 l = []
for k in d.keys() :
l = l + d[k]
return(l )<concatenate>
data['Title'].value_counts()
def boundary_img(r,c):
l = []
for j in range(c):
l.append(np.array([0,j]))
for j in range(r):
l.append(np.array([j,c-1]))
for j in range(c):
l.append(np.array([r-1,j]))
for j in range(r):
l.append(np.array([j,0]))
return(l )<concatenate>
srs_filter_title = data['Title'].value_counts() > round(len(data.index)/100 )
def get_intersection(l1,l2):
l11 = [(x[0],x[1])for x in l1]
l22 = [(x[0],x[1])for x in l2]
l = list(set(l11)&set(l22))
l = [np.array(x)for x in l]
return(l )<concatenate>
data['Title'] = data['Title'].apply(lambda x: 'Misc' if srs_filter_title[x]==False else x )
def check_empty_intersection(l1,l2):
l = get_intersection(l1,l2)
 if len(l)> 0:
return(False)
 else:
return(True )<categorify>
data['Title'].value_counts()
def get_enclosed_area(unit):
r,c = unit.ambient
l = boundary_img(r,c)
 i = 0
count = 0
while check_belongs_to([l[i]],unit.position):
count += 1
i += 1
if i > len(l)-1:
break
if count < 2*c + 2*r - 4:
start_key = l[i]
is_a_boundary = False
else:
is_a_boundary = True
u = fund_unit(1,unit.position,unit.ambient)
 img = concat([u],u.ambient)
 units = get_units(img)
 enclosed_area = []
if is_a_boundary:
for v in units:
if check_belongs_to([v.top_left],[u.top_left]):
enclosed_area += v.position
if len(units)> 2 and not is_a_boundary:
for v in units:
if not check_belongs_to([v.top_left],[u.top_left])and check_empty_intersection(l,v.position):
enclosed_area += v.position
return(enclosed_area )<categorify>
list_clm_to_drop = ['Name', 'PassengerId', 'Ticket', 'Cabin', 'Embarked']
data.drop(list_clm_to_drop, axis=1, inplace=True )
def contained_list(units):
contained = []
enclosed_area = []
for u in units:
if u.colour != 0:
enclosed_area += get_enclosed_area(u)
 for v in units:
if check_belongs_to(v.position,enclosed_area):
contained.append(v)
 return(contained )<prepare_x_and_y>
data = pd.get_dummies(data )
def contained_in(units):
d = {}
l = detect_regular_non_contractible(units)
 for u in l:
d[u] = []
x = u.top_left
y = u.bottom_right
l1 = list(range(x[0],y[0]+1))
l2 = list(range(x[1],y[1]+1))
temp = list(itertools.product(l1,l2))
p = []
for t in temp:
p.append(np.array([t[0],t[1]]))
pdiff = []
for v in p:
r0 = 0
for z in u.position:
if v[0] == z[0] and v[1] == z[1]:
r0 += 1
if r0 == 0:
pdiff.append(v)
 udiff = fund_unit(0,pdiff,u.ambient)
 for v in units:
if check_belongs_to(v.position,u.position):
d[u].append(v)
 return(d )<categorify>
print(data.isnull().sum() )
def get_regular_shapes(units):
d = {}
d['square'] = []
d['vertical line'] = []
d['horizontal line'] = []
d['horizontal rectangle'] = []
d['vertical rectangle'] = []
d['-1'] = []
for u in units:
d[is_regular(u)].append(u)
 return(d )<categorify>
train = data.loc[data['Survived'].isnull() ==False, :]
test = data[-len(test):]
def shape_equivalences(units,return_lists = True,remove_background = False):
units = sort_by_size(units)
 if remove_background:
units = units[:-1]
d = {}
d[units[0]] = []
is_key = {}
for u in units:
is_key[u] = False
for u in units:
c = 0
for v in units:
if is_key[v]:
if len(u.position)== len(v.position):
l = pattern_matching(u,v)
 if len(l)> 0:
c += 1
d[v].append(u)
 if c == 0:
d[u] = [u]
is_key[u] = True
m = len(d.keys())
 shape_equivalences = {}
for j,key in enumerate(d.keys()):
for u in d[key]:
shape_equivalences[u] = j
if return_lists:
return(d)
 else:
return(shape_equivalences )<randomize_order>
train.isnull().sum()
def get_task_num(img,task):
m = len(task['train'])
 for i in range(m):
if np.all(img == task['train'][i]['input']):
return(i )<categorify>
train.isnull().sum()
def common_shapes_across_images(img,task,img_units,num):
m = len(task['train'])
 all_units = []
task_number = {}
for i in range(m):
if i != num:
units = get_units(task['train'][i]['input'])
 else:
units = img_units
for u in units:
task_number[u] = i
all_units += units
if num == -1:
units = img_units
all_units += img_units
for u in units:
task_number[u] = i
d = shape_equivalences(all_units)
 shapes_across_images = []
for x in d.keys() :
l = [task_number[y] for y in d[x]]
if len(set(l)) == m:
shapes_across_images.append(x)
 final_list = []
for x in shapes_across_images:
for u in d[x]:
if task_number[u] == num:
final_list.append(u)
 return(final_list )<categorify>
test.isnull().sum()
def shape_equivalences_counts(units,remove_background = False):
d = {}
s = shape_equivalences(units,True,remove_background)
 for x in s.keys() :
m = len(s[x])
 if m in d.keys() :
d[m] += s[x]
else:
d[m] = s[x]
return(d )<categorify>
test.isnull().sum()
def check_belongs_to(l1,l2):
r = 0
for z in l1:
for x in l2:
if x[0] == z[0] and x[1] == z[1]:
r += 1
if r == len(l1):
return(True)
 else:
return(False )<categorify>
y_train = train['Survived']
X_train = train.drop('Survived', axis = 1)
 X_test = test.drop('Survived', axis = 1 )
def check_this_please(u,x1,x2):
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]]))
if check_belongs_to(l1,u.position):
count = 1
else:
count = 0
return(count )<prepare_x_and_y>
X_test.isnull().sum()
def detect_regular_non_contractible(units):
units = sort_by_size(units)
 regular_non_contractible = []
for u in units:
x = u.top_left
y = u.bottom_right
l1 = list(range(x[0],y[0]+1))
l2 = list(range(x[1],y[1]+1))
temp = list(itertools.product(l1,l2))
p = []
for t in temp:
p.append(np.array([t[0],t[1]]))
pdiff = []
for v in p:
r0 = 0
for z in u.position:
if v[0] == z[0] and v[1] == z[1]:
r0 += 1
if r0 == 0:
pdiff.append(v)
 if len(pdiff)> 0:
udiff = fund_unit(0,pdiff,u.ambient)
 u0 = fund_unit(0,p,u.ambient)
 s = is_regular(udiff)
 if s != '-1':
count = 0
x1 = udiff.top_left
x2 = udiff.top_right
l1 = list(range(x1[1],x2[1]+1))
l2 = []
for t in l1:
l2.append(np.array([x1[0]-1,t]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.top_right
x2 = udiff.bottom_right
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]+1]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.bottom_left
x2 = udiff.bottom_right
l1 = list(range(x1[1],x2[1]+1))
l2 = []
for t in l1:
l2.append(np.array([x1[0]+1,t]))
if check_belongs_to(l2,u.position):
count += 1
x1 = udiff.top_left
x2 = udiff.bottom_left
l1 = list(range(x1[0],x2[0]+1))
l2 = []
for t in l1:
l2.append(np.array([t,x1[1]-1]))
if check_belongs_to(l2,u.position):
count += 1
if count == 4:
regular_non_contractible.append(u)
 return(regular_non_contractible )<categorify>
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from xgboost import XGBClassifier
import lightgbm as lgb
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
def filter_by_size(units):
f = {}
for j in range(1,7):
f[j] = []
for u in units:
if len(u.position)< 6:
f[len(u.position)].append(u)
 else:
f[6].append(u)
 return(f )<concatenate>
list_mlas = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier() ,
lgb.LGBMClassifier()
]
def concat(l,ambient,background = 0):
r,c = ambient
a = np.full(ambient,background)
 for x in l:
for z in x.position:
if z[0] >= 0 and z[0] <= r-1 and z[1] >= 0 and z[1] <= c-1:
a[z[0],z[1]] = x.colour
return(a )<categorify>
obj_splitter_for_cv = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 42 )
def input_output_shape_is_same(task):
return all([np.array(el['input'] ).shape == np.array(el['output'] ).shape for el in task['train']])
 def get_equals(task_series):
l = []
for j in task_series.index:
if input_output_shape_is_same(task_series[j]):
l.append(j)
 return(l )<sort_values>
list_mla_columns = ['ml_name', 'parameters','train_accuracy_mean', 'test_accuracy_mean', 'test_accuracy_3sigma' ,'elapsed_time']
df_result_in_each_mla = pd.DataFrame(columns = list_mla_columns)
 df_result_prediction = pd.DataFrame(y_train, columns=['answer'] )
l_input_output_same_size = get_equals(train_tasks)
 l_input_output_same_size.sort()<categorify>
int_index_mla_list = 0
for alg in list_mlas:
str_ml_name = alg.__class__.__name__
df_result_in_each_mla.loc[int_index_mla_list, 'ml_name'] = str_ml_name
df_result_in_each_mla.loc[int_index_mla_list, 'parameters'] = str(alg.get_params())
 dict_cv_result = model_selection.cross_validate(alg, X_train, y_train, cv=obj_splitter_for_cv, return_train_score=True)
 df_result_in_each_mla.loc[int_index_mla_list, 'elapsed_time'] = dict_cv_result['fit_time'].mean()
df_result_in_each_mla.loc[int_index_mla_list, 'train_accuracy_mean'] = dict_cv_result['train_score'].mean()
df_result_in_each_mla.loc[int_index_mla_list, 'test_accuracy_mean'] = dict_cv_result['test_score'].mean()
df_result_in_each_mla.loc[int_index_mla_list, 'test_accuracy_3sigma'] = dict_cv_result['test_score'].std() *3
alg.fit(X_train, y_train)
 df_result_prediction[str_ml_name] = alg.predict(X_train)
 int_index_mla_list += 1
def convert_to_tuple(t):
s = [(a[0],a[1])for a in t]
return(s )<categorify>
df_result_in_each_mla.sort_values(by = ['test_accuracy_mean'], ascending = False, inplace = True)
 df_result_in_each_mla
l_colour_match_no_diag = []
for index in l_input_output_same_size:
task = train_tasks[index]
c = 0
for example in task['train']:
units_input = get_units(np.array(example['input']))
units_output = get_units(np.array(example['output']))
if len(units_input)== len(units_output):
sub_count = 0
for u in units_input:
s1 = set(convert_to_tuple(u.position))
for v in units_output:
s2 = set(convert_to_tuple(v.position))
if s1 == s2:
sub_count += 1
if sub_count == len(units_input):
c += 1
if c == len(task['train']):
l_colour_match_no_diag.append(index )<categorify>
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=0.3,
random_state=42,
stratify=y_train )
def get_lists(img,task = None,num = -1,with_diag = False):
list_dict = {}
img = np.array(img)
 ambient = np.array(img ).shape
units = get_units(img,with_diag)
 list_dict[0] = units
list_dict[1] = get_non_contractible(units,with_diag,False)
 list_dict[2] = surrounded_list(units)
 list_dict[3] = contained_list(units)
 shape_d = get_regular_shapes(units)
 list_dict[4] = shape_d['square']
list_dict[5] = shape_d['vertical line']
list_dict[6] = shape_d['horizontal line']
list_dict[7] = shape_d['horizontal rectangle']
list_dict[8] = shape_d['vertical rectangle']
list_dict[9] = shape_d['-1']
list_dict[10] = list_dict[4] + list_dict[5] + list_dict[6] + list_dict[7] + list_dict[8]
list_dict[11] = detect_regular_non_contractible(units)
 shape_counts = shape_equivalences_counts(units,True)
 for i in range(1,4):
if i not in shape_counts.keys() :
shape_counts[i] = []
for i in range(1,4):
list_dict[11+i] = shape_counts[i]
list_dict[15] = common_shapes_across_images(img,task,units,num)
 f = filter_by_size(units)
 for j in range(1,7):
list_dict[15+j] = f[j]
list_dict = get_list_dict_complements(units,list_dict)
 return(list_dict )<categorify>
scoring = 'f1'
dict_param_grid = {
'num_leaves' : [4, 9, 16, 25],
'max_depth' : [2, 3, 4, 5],
'colsample_bytree' : [0.5, 1.0],
'subsample': [0.2, 0.6, 1.0],
'learning_rate' : [0.01, 0.1]
}
obj_gridsearch_cv = linear_model.RidgeClassifierCV(alphas=[1e-3, 1e-2, 1e-1, 1])
 obj_gridsearch_cv.fit(X_train, y_train)
def get_list_dict_complements(units,list_dict):
e = {}
for i in list_dict.keys() :
if i > 0:
t = list(set(units)- set(list_dict[i]))
e[-i] = t
for j in e.keys() :
list_dict[j] = e[j]
return(list_dict )<define_variables>
y_train_pred = obj_gridsearch_cv.predict(X_train)
 y_test_pred = obj_gridsearch_cv.predict(X_test)
 ', obj_gridsearch_cv.best_params_)
 print('Scores')
 print('Close: {:.3f}'.format(obj_gridsearch_cv.score(X_train, y_train)))
 print('Open: {:.3f}'.format(obj_gridsearch_cv.score(X_valid, y_valid)) )
def extract_lists_from_dict(d):
t = []
for x in d.keys() :
t.append(d[x])
 return(t )<categorify>
sub = pd.DataFrame(pd.read_csv(".. /input/titanic/test.csv")['PassengerId'])
 sub['Survived'] = list(map(int, y_test_pred))
sub.to_csv("submission.csv", index=False )