Kaggle competition: Titanic - Machine Learning from Disaster
def move_in_evens(l):
m = len(l)
 if m > 0:
r = []
c = 0
while c < m:
r.append(c)
 c += 2
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )<normalization>
train_dir = ".. /input/train.csv"
test_dir = ".. /input/test.csv"
def move_in_odds(l):
m = len(l)
 if m > 0:
r = []
c = 1
while c < m:
r.append(c)
 c += 2
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )<categorify>
df = pd.read_csv(train_dir)
 test_df = pd.read_csv(test_dir)
 print("Total number of instance : ",len(df))
df.isna().sum()
def move_in_3s(l):
m = len(l)
 if m > 0:
r = []
c = 0
while c < m:
r.append(c)
 c += 3
l = np.array(l)
 l = list(l[r])
 return(l)
 else:
return([] )<categorify>
df.drop(["Cabin"], axis = 1, inplace = True)
 test_df.drop(["Cabin"], axis = 1, inplace = True)
 df.drop(["Ticket"], axis = 1, inplace = True)
 test_df.drop(["Ticket"], axis = 1, inplace = True)
 df.info()
df.head(10 )
def middle(l):
m = len(l)
 if m > 0:
if m%2 == 0:
return([l[m//2]])
 if m%2 == 1:
t =(m-1)//2
return([l[t]])
 else:
return([] )<categorify>
df.drop("PassengerId", axis = 1, inplace = True)
 test_df.drop("PassengerId", axis = 1, inplace = True )
def start_mid_end(l):
if len(l)> 0:
mid = middle(l)[0]
return([l[0],mid,l[-1]])
 else:
return([] )<categorify>
df["Sex"].replace("male", 0, inplace = True)
 test_df["Sex"].replace("male", 0, inplace = True)
 df["Sex"].replace("female", 1, inplace = True)
 test_df["Sex"].replace("female", 1, inplace = True)
 df["Embarked"].replace(["S","C","Q"],[0,1,2], inplace = True)
 test_df["Embarked"].replace(["S","C","Q"],[0,1,2], inplace = True )
def template_sublist(l,template):
d = {}
m = len(l)
 for i in range(4):
if m > i:
d[i] = [l[i]]
else:
d[i] = []
for i in range(1,5):
if m >= i:
d[3+i] = [l[-i]]
else:
d[3+i] = []
d[8]= l[1:]
d[9]= l[2:]
d[10]= l[3:]
d[11]= l[:-1]
d[12]= l[:-2]
d[13]= l[:-3]
d[14] = move_in_evens(l)
 d[15] = move_in_odds(l)
 d[16] = move_in_3s(l)
 d[17] = middle(l)
 d[18] = start_mid_end(l)
 d[19] = l
return(d[template] )<sort_values>
def create_family_ranges(df):
familysize = []
for members in df["n_fam_mem"]:
if members == 0:
familysize.append(0)
 elif members > 0 and members <=4:
familysize.append(1)
 elif members > 4:
familysize.append(2)
 return familysize
famsize = create_family_ranges(df)
 df["familysize"] = famsize
test_famsize = create_family_ranges(test_df)
 test_df["familysize"] = test_famsize
def sorting_list(l,sorting_type):
if sorting_type <= 7:
return(sort_by_position(l,sorting_type))
if sorting_type == 8:
return(sort_by_size(l))<categorify>
def age_to_int(df):
agelist = df["Age"].values.tolist()
for i in range(len(agelist)) :
if agelist[i] < 18 and agelist[i] >= 0:
agelist[i] = 0
elif agelist[i] >= 18 and agelist[i] < 60:
agelist[i] = 1
elif agelist[i]>=60 and agelist[i]<200:
agelist[i] = 2
else:
agelist[i] = -1
ageint = pd.DataFrame(agelist)
 return ageint
def get_list_selection(l,sorting_type,template,colour_filter):
l = sorting_list(l,sorting_type)
 l = template_sublist(l,template)
 c_dict = get_colour_dict(l)
 if colour_filter < 10:
t = c_dict[colour_filter]
else:
t = l
return(t )<prepare_x_and_y>
ageint = age_to_int(df)
 df["Ageint"] = ageint
df.drop("Age", axis = 1, inplace = True)
 test_ageint = age_to_int(test_df)
 test_df["Ageint"] = test_ageint
test_df.drop("Age", axis = 1, inplace = True)
def is_equal(l1,l2):
count1 = 0
count2 = 0
m1 = len(l1)
 m2 = len(l2)
 if m1 == 0 and m2 > 0:
return(False)
 if m1 > 0 and m2 == 0:
return(False)
 for u in l1:
for v in l2:
x = [(a[0],a[1])for a in u.position]
y = [(b[0],b[1])for b in v.position]
if set(x)== set(y)and u.colour == v.colour:
count1 +=1
for u in l2:
for v in l1:
x = [(a[0],a[1])for a in u.position]
y = [(b[0],b[1])for b in v.position]
if set(x)== set(y)and u.colour == v.colour:
count2 +=1
if count1 == len(l1)and count2 == len(l2):
return(True)
 else:
return(False )<normalization>
def conv_fare_ranges(df):
fare_ranges = []
for fare in df.actual_fare:
if fare < 7:
fare_ranges.append(0)
 elif fare >=7 and fare < 14:
fare_ranges.append(1)
 elif fare >=14 and fare < 30:
fare_ranges.append(2)
 elif fare >=30 and fare < 50:
fare_ranges.append(3)
 elif fare >=50:
fare_ranges.append(4)
 return fare_ranges
fare_ranges = conv_fare_ranges(df)
 df["fare_ranges"] = fare_ranges
test_fare_ranges = conv_fare_ranges(test_df)
 test_df["fare_ranges"] = test_fare_ranges
def get_colour_change_dict(task,task_number):
img_in = task['train'][task_number]['input']
img_out = task['train'][task_number]['output']
img_in = np.array(img_in)
 ambient = img_in.shape
img_out = np.array(img_out)
 units_in = sort_by_position(get_units(img_in),0)
 units_out = sort_by_position(get_units(img_out),0)
 m = len(units_in)
 colour_diffs = {}
for i in range(m):
if units_in[i].colour != units_out[i].colour:
if units_out[i].colour not in colour_diffs.keys() :
colour_diffs[units_out[i].colour] = [units_in[i]]
else:
colour_diffs[units_out[i].colour].append(units_in[i])
 return(colour_diffs )<merge>
def name_to_int(df):
name = df["Name"].values.tolist()
namelist = []
for i in name:
index = 1
inew = i.split()
if inew[0].endswith(","):
index = 1
elif inew[1].endswith(","):
index = 2
elif inew[2].endswith(","):
index = 3
namelist.append(inew[index])
 print(set(namelist))
titlelist = []
for i in range(len(namelist)) :
if namelist[i] == "Lady.":
titlelist.append("Lady.")
 elif namelist[i] == "Ms.":
titlelist.append("Ms.")
 elif namelist[i] == "Miss.":
titlelist.append("Miss.")
 elif namelist[i] == "Dr.":
titlelist.append("Dr.")
 elif namelist[i] == "Mr.":
titlelist.append("Mr.")
 elif namelist[i] == "Jonkheer.":
titlelist.append("Jonkheer.")
 elif namelist[i] == "Col.":
titlelist.append("Col.")
 elif namelist[i] == "Mrs.":
titlelist.append("Mrs")
 elif namelist[i] == "Sir.":
titlelist.append("Sir.")
 elif namelist[i] == "Mlle.":
titlelist.append("Mlle.")
 elif namelist[i] == "Capt.":
titlelist.append("Capt.")
 elif namelist[i] == "the":
titlelist.append("the")
 elif namelist[i] == "Don.":
titlelist.append("Don.")
 elif namelist[i] == "Master.":
titlelist.append("Master.")
 elif namelist[i] == "Rev.":
titlelist.append("Rev.")
 elif namelist[i] == "Mme.":
titlelist.append("Mme.")
 elif namelist[i] == "Major.":
titlelist.append("Major.")
 else:
titlelist.append("sometitle")
 print(set(namelist))
return titlelist
def intersection(lst1, lst2):
return list(set(lst1)& set(lst2))<statistical_test>
titlelist = name_to_int(df)
 df["titles"] = titlelist
df["titles"].value_counts()
testtitlelist = name_to_int(test_df)
 test_df["titles"] = testtitlelist
def whittle_candidates_down(task,task_number,colour_diff_solutions,with_diag = False):
img = np.array(task['train'][task_number]['input'])
 list_dict = get_lists(img,task,task_number,with_diag)
 colour_diff = get_colour_change_dict(task,task_number)
 for c in colour_diff.keys() :
if c in colour_diff_solutions.keys() :
temp = colour_diff_solutions[c]
else:
temp = []
p = len(temp)
 if p == 0:
img = np.array(task['train'][task_number]['input'])
 colour_diff_solutions[c] = identify_generic_list(colour_diff[c],img,task,task_number,with_diag)
 else:
temp1 = []
for i in range(p):
a,b,c1,d = temp[i]
t = get_list_selection(list_dict[a],b,c1,d)
 if is_equal(t,colour_diff[c]):
temp1.append(temp[i])
 colour_diff_solutions[c] = temp1
e = get_unique_dict_values(colour_diff_solutions)
 return(e )<init_hyperparams>
df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 test_df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].value_counts()
def training_algo(task,with_diag = False):
m = len(task['train'])
 colour_diff = get_colour_change_dict(task,0)
 img = np.array(task['train'][0]['input'])
 colour_diff_solutions = {}
for c in colour_diff.keys() :
l_c = colour_diff[c]
solutions = identify_generic_list(l_c,img,task,0,with_diag)
 colour_diff_solutions[c] = solutions
for i in range(1,m):
colour_diff_solutions = whittle_candidates_down(task,i,colour_diff_solutions,with_diag)
 for i in range(m-1,-1,-1):
colour_diff_solutions = whittle_candidates_down(task,i,colour_diff_solutions,with_diag)
 return(colour_diff_solutions )<predict_on_test>
df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 df["titles"].astype("int64")
 test_df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 test_df["titles"].astype("int64")
 df.drop(["Name"], axis = 1, inplace = True)
 test_df.drop(["Name"], axis = 1, inplace = True )
def collect_predictions(task,with_diag = False):
colour_diff_solutions = training_algo(task,with_diag)
 preds = []
m = len(task['test'])
 for i in range(m):
img_out = assemble_test(task,i,colour_diff_solutions,with_diag)
 preds.append(img_out)
 return(preds )<categorify>
df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True)
 test_df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True )
def assemble_test(task,task_number,colour_diff_solutions,with_diag = False):
img = np.array(task['test'][task_number]['input'])
 ambient = img.shape
list_dict = get_lists(img,task,-1,with_diag)
 output_units = []
for c in colour_diff_solutions.keys() :
if len(colour_diff_solutions[c])> 0:
temp = filter_solutions(list_dict,colour_diff_solutions,c)
 else:
temp = []
for t in temp:
t.colour = c
output_units += temp
units = get_list_selection(list_dict[0],0,19,10)
 other_units = []
m = len(output_units)
 if m == 0:
other_units = units
else:
for u in units:
c = 0
for v in output_units:
if not is_equal([u],[v]):
c += 1
if c == m:
other_units.append(u)
 output_units = other_units + output_units
img_out = concat(output_units,ambient)
 return(img_out )<feature_engineering>
labels = df["Survived"]
data = df.drop("Survived", axis = 1)
 X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1 )
def filter_solutions(list_dict,colour_diff_solutions,c):
e = {}
for x in colour_diff_solutions[c]:
s = get_list_selection(list_dict[x[0]],x[1],x[2],x[3])
 s = sort_by_position(s,0)
 s = tuple(s)
 if s in e.keys() :
e[s] += 1
else:
e[s] = 1
l = list(e.keys())
 l.sort(key = lambda x : e[x])
 t = list(l[-1])
 return(t )<split>
final_clf = None
clf_names = ["Logistic Regression", "KNN(3)", "KNN(5)", "Random forest classifier", "Decision Tree Classifier",
"Gradient Boosting Classifier", "Support Vector Machine"]
classifiers = []
scores = []
task = train_tasks[l_colour_match_no_diag[31]]
img = task['train'][0]['input']<categorify>
bestknn5 = None
bestknn3 = None
bestrf = None
bestgb = None
bestcvm = None
bestlr = None
bestdt = None
for i in range(10):
X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1)
 tempscores = []
lr_clf = LogisticRegression()
lr_clf.fit(X_train, Y_train)
 tempscores.append(( lr_clf.score(X_test, Y_test)) *100)
 if lr_clf.score(X_test, Y_test)*100 > 85:
bestlr = lr_clf
knn3_clf = KNeighborsClassifier()
knn3_clf.fit(X_train, Y_train)
 tempscores.append(( knn3_clf.score(X_test, Y_test)) *100)
 if knn3_clf.score(X_test, Y_test)*100 > 85:
bestknn3 = knn3_clf
knn5_clf = KNeighborsClassifier()
knn5_clf.fit(X_train, Y_train)
 tempscores.append(( knn5_clf.score(X_test, Y_test)) *100)
 if knn5_clf.score(X_test, Y_test)*100 > 85:
bestknn5 = knn5_clf
rf_clf = RandomForestClassifier(n_estimators = 100)
 rf_clf.fit(X_train, Y_train)
 tempscores.append(( rf_clf.score(X_test, Y_test)) *100)
 if rf_clf.score(X_test, Y_test)*100 > 85:
bestrf = rf_clf
dt_clf = DecisionTreeClassifier()
dt_clf.fit(X_train, Y_train)
 tempscores.append(( dt_clf.score(X_test, Y_test)) *100)
 if dt_clf.score(X_test, Y_test)*100 > 85:
bestdt = dt_clf
gb_clf = GradientBoostingClassifier()
gb_clf.fit(X_train, Y_train)
 tempscores.append(( gb_clf.score(X_test, Y_test)) *100)
 if gb_clf.score(X_test, Y_test)*100 > 85:
bestgb = lr_clf
svm_clf = SVC(gamma = "scale")
 svm_clf.fit(X_train, Y_train)
 tempscores.append(( svm_clf.score(X_test, Y_test)) *100)
 if svm_clf.score(X_test, Y_test)*100 > 85:
bestsvm = svm_clf
scores.append(tempscores )
def check_colour_match(task):
c = 0
for example in task['train']:
units_input = get_units(np.array(example['input']))
units_output = get_units(np.array(example['output']))
if len(units_input)== len(units_output):
sub_count = 0
for u in units_input:
s1 = set(convert_to_tuple(u.position))
for v in units_output:
s2 = set(convert_to_tuple(v.position))
if s1 == s2:
sub_count += 1
if sub_count == len(units_input):
c += 1
if c == len(task['train']):
return(True)
 else:
return(False )<categorify>
scores = np.array(scores)
 clfs = pd.DataFrame({"Classifier":clf_names})
 clfs["iteration0"] = scores[0].T
clfs["iteration1"] = scores[1].T
clfs["iteration2"] = scores[2].T
clfs["iteration3"] = scores[3].T
clfs["iteration4"] = scores[4].T
clfs["iteration5"] = scores[5].T
clfs["iteration6"] = scores[6].T
clfs["iteration7"] = scores[7].T
clfs["iteration8"] = scores[8].T
clfs["iteration9"] = scores[9].T
means = clfs.mean(axis = 1)
 means = means.values.tolist()
clfs["Average"] = means
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def make_prediction(tasks):
result = pd.Series()
for idx, task in tqdm(tasks.iteritems()):
if input_output_shape_is_same(task)and check_colour_match(task):
pred = collect_predictions(task)
 else:
pred = [el['input'] for el in task['test']]
for i, p in enumerate(pred):
result[f'{idx}_{i}'] = flattener(np.array(p ).tolist())
 return result<predict_on_test>
final_clf = bestsvm
submission = make_prediction(test_tasks)
 submission.head()<save_to_csv>
test_data = test_df
predictions = final_clf.predict(test_data)
 print(len(predictions))
submission = submission.reset_index()
submission.columns = ['output_id', 'output']
submission.to_csv('submission.csv', index=False)
 submission<import_modules>
final_csv = []
csv_title = ['PassengerId', 'Survived']
final_csv.append(csv_title)
 for i in range(len(predictions)) :
passengerid = i + 892
survived = predictions[i]
temp = [passengerid, survived]
final_csv.append(temp)
 print(len(final_csv))
with open('submission_csv.csv', 'w')as file:
writer = csv.writer(file)
 writer.writerows(final_csv)
 file.close()