Kaggle competition: Titanic - Machine Learning from Disaster
show_hit_dist(s_hidden, results, k=k_top )<compute_test_metric>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train_data.info()
print("Model: Hybrid Collaborative Filtering")
 print("Similarity: Hybrid(0.2 * Jaccard + 0.8 * Cosine)")
 collab_hybrid = weighted_hybrid([ 
(collab_jac10, 0.2), 
(collab_cos10, 0.8)
 ])
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_hybrid), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_hybrid), k=k_top)) )<import_modules>
missing_train_total = train_data.isnull().sum().sort_values(ascending= False)
 missing_train_percentage =(train_data.isnull().sum() /train_data.count() ).sort_values(ascending= False)
 missing_train_data = pd.concat([missing_train_total, missing_train_percentage], axis=1, keys=['Total', 'Percent'])
 missing_train_data.head(10 )
from redcarpet import write_kaggle_recs, download_kaggle_recs<load_pretrained>
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_percentage =(test_data.isnull().sum() /test_data.count() ).sort_values(ascending= False)
 missing_test_data = pd.concat([missing_test_total, missing_test_percentage], axis=1, keys=['Total', 'Percent'])
 missing_test_data.head(10 )
s_hold_input = pickle.load(open(".. /input/hold_set.pkl", "rb"))
print("Hold Out Set: N = {}".format(len(s_hold_input)))
 s_all_input = s_input + s_hold_input
print("All Input: N = {}".format(len(s_all_input)) )<choose_model_class>
train_data.groupby('Pclass')['Age'].mean()
print("Final Model")
 print("Strategy: Collaborative")
 print("Similarity: Cosine(j=10)")
 final_scores = collaborative_filter(s_train, s_all_input, sim_fn=cosine_sim, j=41, threshold=0.08, k=10)
 final_recs = get_recs(final_scores )<save_to_csv>
train_data.groupby(['Pclass','Sex'])['Sex'].count()
outfile = "kaggle_submission_hybrid_collab.csv"
n_lines = write_kaggle_recs(final_recs, outfile)
 print("Wrote predictions for {} users to {}.".format(n_lines, outfile))
download_kaggle_recs(final_recs, outfile )<import_modules>
train_data.groupby('Sex')['Sex'].count()
import pandas as pd
import numpy as np
import sklearn
import os
from matplotlib import pyplot as plt<define_variables>
mean = train_data.groupby(['Pclass','Sex'])['Age'].mean()
median = train_data.groupby(['Pclass','Sex'])['Age'].median()
age_sex_Pclass = pd.concat([mean, median], axis=1, keys=['Age mean', 'Age median'])
 age_sex_Pclass.head(6)
filepath = ".. /input/adult-dataset/train_data.csv"<load_from_csv>
train_data.Embarked.value_counts()
adult = pd.read_csv(filepath,
names=[
"Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
sep=r'\s*,\s*',
engine='python',
na_values="?" )<count_missing_values>
features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
X = train_data[features]
y = train_data.Survived
Pclass_Sex_Age_median = X.groupby(['Pclass','Sex'] ).Age.transform('median')
 X.Age.fillna(Pclass_Sex_Age_median, inplace = True)
 Pclass_Fare_median = X.groupby('Pclass' ).Fare.transform('median')
 X.Fare.fillna(Pclass_Fare_median, inplace = True)
 missing_test_total = X.isnull().sum().sort_values(ascending= False)
 X.Embarked.fillna('S', inplace = True)
 X['Embarked'].replace({'S': 0, 'Q': 1, 'C': 2}, inplace = True)
 X['Sex'].replace({'female': 0, 'male': 1}, inplace = True)
 X.info()
print(X.Sex )
listNull = [x for x in [(row,adult[row].isnull().sum())for row in adult] if x[1]!=0]
print(listNull )<drop_column>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.25, random_state=1)
 print(type(X_train))
adult.drop(adult.index[0], inplace=True )<count_values>
clf_mae=[]
for md in range(8):
clf = RandomForestClassifier(max_depth=(md+1), random_state=1)
 clf.fit(X_train, y_train)
 predictions = clf.predict(X_val)
 mae =(1.- mean_absolute_error(predictions, y_val)) *100.
 clf_mae.append(mae)
 print(clf_mae )
adult["Workclass"].value_counts()
adult["Occupation"].value_counts() 
<define_variables>
clf = RandomForestClassifier(max_depth=4, random_state=1)
 clf.fit(X_train, y_train)
 predictions = clf.predict(X_val)
 mae =(1.- mean_absolute_error(predictions, y_val)) *100.
 print(mae )
firstW = ["United-States",
"Germany",
"Canada",
"England",
"Italy",
"Japan",
"Poland",
"Portugal",
"Taiwan",
"France",
"Greece",
"Ireland",
"Hong",
"Yugoslavia",
"Hungary",
"Scotland",
"Holand-Netherlands"]<categorify>
logreg = LogisticRegression(solver='liblinear')
 logreg.fit(X_train, y_train)
 predictions = logreg.predict(X_val)
 mae = mean_absolute_error(predictions, y_val)
 print(( 1-mae)*100,"%")
adult["Race"] = adult["Race"].transform(lambda x: 1 if x=="White" else 0 if x==x else x)
 adult["Country"] = adult["Country"].transform(lambda x: 1 if x in firstW else -1 if x == "Mexico" else 0 if x==x else x)
 adult["Sex"].replace({"Male":1,"Female":0}, inplace=True )<feature_engineering>
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_total.head()
adult["Age"] = adult["Age"].transform(lambda x: int(x))
adult["Age"] = adult["Age"].transform(lambda x: 2*(x - x.mean())/ x.std())
 adult["Workclass"] = adult["Workclass"].transform(lambda x: 1 if x in ["Without-pay", "Never-worked"] else 0 if x==x else x)
 adult["Occupation"] = adult["Occupation"].transform(lambda x: 1 if x in ["Priv-house-serv","Handlers-cleaners","Transport-moving"] else 0 if x==x else x)
<feature_engineering>
Pclass_Sex_Age_median = test_data.groupby(['Pclass','Sex'] ).Age.transform('median')
 test_data.Age.fillna(Pclass_Sex_Age_median, inplace = True)
 Pclass_Fare_median = test_data.groupby('Pclass' ).Fare.transform('median')
 test_data.Fare.fillna(Pclass_Fare_median, inplace = True)
 test_data['Embarked'].replace({'S': 0, 'Q': 1, 'C': 2}, inplace = True)
 test_data['Sex'].replace({'female': 0, 'male': 1}, inplace = True)
adult["MissingCountry"] = adult["Country"].transform(lambda x: 1 if x==x else 0)
 adult["Country"].fillna(adult["Country"].mean() , inplace=True)
 adult["MissingWorkclass"] = adult["Workclass"].transform(lambda x: 1 if x==x else 0)
 adult["Workclass"].fillna(adult["Workclass"].mean() , inplace=True)
 adult["MissingOccupation"] = adult["Occupation"].transform(lambda x:1 if x==x else 0)
 adult["Occupation"].fillna(adult["Occupation"].mean() ,inplace=True )<correct_missing_values>
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_total.head()
nadult = adult.dropna()<count_values>
nadult["MissingCountry"].value_counts()<define_variables>
name_list_train = train_data.Name.to_list()
print(name_list_train )
testfilepath = ".. /input/adult-dataset/test_data.csv"<load_from_csv>
list_of_titles = title_retriever(name_list_train)
 print(len(list_of_titles))
train_data['Title'] = list_of_titles
train_data.Title.value_counts()
testAdult = pd.read_csv(testfilepath,
names=[
"Id","Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
sep=r'\s*,\s*',
engine='python',
na_values="?" )<count_values>
Title_Age_median = train_data.groupby(['Title'] ).Age.transform('median')
 train_data.Age.fillna(Title_Age_median, inplace = True)
testAdult
testAdult["Workclass"].value_counts()<drop_column>
name_list_test = test_data.Name.to_list()
print(name_list_test )
testAdult.drop(testAdult.index[0], inplace=True )<drop_column>
list_of_titles_test = title_retriever(name_list_test)
 print(len(list_of_titles_test))
test_data['Title'] = list_of_titles_test
test_data.Title.value_counts()
testAdult.drop("Id",axis=1, inplace=True)
 testAdult.drop("Target",axis=1, inplace=True )<count_missing_values>
Title_Age_median_test = test_data.groupby(['Title'] ).Age.transform('median')
 test_data.Age.fillna(Title_Age_median_test, inplace = True)
listNull = [x for x in [(row,testAdult[row].isnull().sum())for row in testAdult] if x[1]!=0]
print(listNull )<feature_engineering>
X = train_data[features]
y = train_data.Survived
Pclass_Fare_median = X.groupby('Pclass' ).Fare.transform('median')
 X.Fare.fillna(Pclass_Fare_median, inplace = True)
 X.Embarked.fillna('S', inplace = True)
 X['Embarked'].replace({'S': 0, 'Q': 1, 'C': 2}, inplace = True)
 X['Sex'].replace({'female': 0, 'male': 1}, inplace = True)
 X.info()
print(X.Sex)
testAdult["Race"] = testAdult["Race"].transform(lambda x: 1 if x=="White" else 0)
 testAdult["Country"] = testAdult["Country"].transform(lambda x: 1 if x in firstW else -1 if x=="Mexico" else 0)
 testAdult["Sex"] = testAdult["Sex"].transform(lambda x: 1 if x=="Male" else 0)
 testAdult["Age"] = testAdult["Age"].transform(int)
 testAdult["Age"] = testAdult["Age"].transform(lambda x: 2*(x - x.mean())/ x.std())
 testAdult["Workclass"] = testAdult["Workclass"].transform(lambda x: 1 if x in ["Without-pay", "Never-worked"] else 0)
 testAdult["Occupation"] = testAdult["Occupation"].transform(lambda x: 1 if x in ["Priv-house-serv","Handlers-cleaners","Transport-moving"] else 0)
<categorify>
X_test = test_data[features]
clf.fit(X, y)
 predictions = clf.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission.csv', index=False)
 print("Your submission was successfully saved!" )