Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_unique_values>
%matplotlib inline
plt.style.use('fivethirtyeight')
train_df.nunique()<count_unique_values>
df_train = pd.read_csv(".. /input/titanic/train.csv", header=0)
 df_train.head()
test_df.nunique()<concatenate>
df_test = pd.read_csv(".. /input/titanic/test.csv", header=0)
 df_test.head()
concatenated = pd.concat([train_df, test_df])
 concatenated.shape<count_unique_values>
print("Training data size:{}".format(df_train.shape))
print("Test data size:{}".format(df_test.shape))
concatenated.nunique()<count_values>
df_train.isnull().sum()
th10 = pd.DataFrame(train_df.landmark_id.value_counts().head(10))
th10.reset_index(level=0, inplace=True)
 th10.columns = ['landmark_id','count']
th10<create_dataframe>
df_test.isnull().sum()
tb10 = pd.DataFrame(train_df.landmark_id.value_counts().tail(10))
tb10.reset_index(level=0, inplace=True)
 tb10.columns = ['landmark_id','count']
tb10<feature_engineering>
df_train["Ticket"].value_counts()
ll = list()
for path in train_df['url']:
ll.append(( path.split('//', 1)[1] ).split('/', 1)[0])
 train_df['site'] = ll
ll = list()
for path in test_df['url']:
ll.append(( path.split('//', 1)[1] ).split('/', 1)[0])
 test_df['site'] = ll<create_dataframe>
df_train["Cabin"].value_counts()
train_site = pd.DataFrame(train_df.site.value_counts())
 test_site = pd.DataFrame(test_df.site.value_counts() )<save_to_csv>
df_train["Age"].fillna(np.mean(df_train["Age"]), inplace=True)
 df_train["Fare"].fillna(np.mean(df_train["Fare"]), inplace=True)
 df_train["Embarked"].fillna('S', inplace=True)
 df_train.dropna(axis=1, inplace=True)
 df_test["Age"].fillna(np.mean(df_train["Age"]), inplace=True)
 df_test["Fare"].fillna(np.mean(df_train["Fare"]), inplace=True)
 df_test.dropna(axis=1, inplace=True )
freq_label = train_df['landmark_id'].value_counts() /train_df['landmark_id'].value_counts().sum()
submission['landmarks'] = '%d %2.2f' %(freq_label.index[0], freq_label.values[0])
 submission.to_csv('submission.csv', index=False)
 np.random.seed(2018)
 r_idx = lambda : np.random.choice(freq_label.index, p = freq_label.values)
 r_score = lambda idx: '%d %2.4f' %(freq_label.index[idx], freq_label.values[idx])
 submission['landmarks'] = submission.id.map(lambda _: r_score(r_idx()))
submission.to_csv('rand_submission.csv', index=False )<import_modules>
def sex(x):
if x["Sex"] == "male":
res = 0
else :
res = 1
return res
df_train["Sex_cate"] = df_train.apply(sex, axis=1)
 df_test["Sex_cate"] = df_test.apply(sex, axis=1)
 def age_band(x):
if x["Age"] <= 10:
res = 0
elif x["Age"] <= 20 and x["Age"] > 10:
res = 1
elif x["Age"] <= 30 and x["Age"] > 20:
res = 2
elif x["Age"] <= 40 and x["Age"] > 30:
res = 3
elif x["Age"] <= 50 and x["Age"] > 40:
res = 4
elif x["Age"] <= 60 and x["Age"] > 50:
res = 5
else :
res = 6
return res
df_train["Age_band"] = df_train.apply(age_band, axis=1)
 df_test["Age_band"] = df_test.apply(age_band, axis=1)
 def fare_band(x):
if x["Fare"] <= 25:
res = 0
elif x["Fare"] <= 50 and x["Fare"] > 25:
res = 1
elif x["Fare"] <= 75 and x["Fare"] > 50:
res = 2
elif x["Fare"] <= 100 and x["Fare"] > 75:
res = 3
elif x["Fare"] <= 125 and x["Fare"] > 100:
res = 4
else :
res = 5
return res
df_train["Fare_band"] = df_train.apply(fare_band, axis=1)
 df_test["Fare_band"] = df_test.apply(fare_band, axis=1)
 def embarked_flg(x):
if x["Embarked"] == 'S':
res = 0
elif x["Embarked"] == 'C':
res = 1
else:
res = 2
return res
df_train["Embarked_flg"] = df_train.apply(embarked_flg, axis=1)
 df_test["Embarked_flg"] = df_test.apply(embarked_flg, axis=1 )
import pandas as pd
import numpy as np<import_modules>
X = df_train[['Pclass','SibSp','Parch','Sex_cate', 'Age_band', 'Fare_band', 'Embarked_flg']]
y = df_train[['Survived']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
 sc = StandardScaler()
sc.fit(X_train)
 X_train_std = sc.transform(X_train)
 X_test_std = sc.transform(X_test )
import pandas as pd
import numpy as np<load_from_csv>
lr = LogisticRegression()
param_range = [0.001, 0.01, 0.1, 1.0]
penalty = ['l1', 'l2']
param_grid = [{"C":param_range, "penalty":penalty}]
gs_lr = GridSearchCV(estimator=lr, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=-1)
 gs_lr = gs_lr.fit(X_train_std, y_train)
 print(gs_lr.best_score_.round(3))
print(gs_lr.best_params_ )
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' )<rename_columns>
svm = SVC(random_state=10, probability=True)
 param_range = [0.001, 0.01, 0.1, 1.0]
param_grid = [{'C':param_range, 'kernel':['linear']}]
gs_svm = GridSearchCV(estimator=svm, param_grid=param_grid, scoring='accuracy', cv=10, n_jobs=-1)
 gs_svm = gs_svm.fit(X_train_std, y_train)
 print(gs_svm.best_score_.round(3))
print(gs_svm.best_params_ )
train.columns = ['date', 'date_block_num', 'shop_id', 'item_id', 'item_price', 'item_cnt_day']
train<sort_values>
knn = KNeighborsClassifier(metric='minkowski')
 param_range = [5, 10, 15, 20]
param_grid = [{"n_neighbors":param_range, "p":[1,2]}]
gs_knn = GridSearchCV(estimator=knn, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=-1)
 gs_knn = gs_knn.fit(X_train_std, y_train)
 print(gs_knn.best_score_.round(3))
print(gs_knn.best_params_ )
train[train.date_block_num==0].sort_values(['shop_id','item_id'], ascending=True )<feature_engineering>
tree = DecisionTreeClassifier(max_depth=4, random_state=10)
 param_range = [3, 6, 9, 12]
leaf = [10, 15, 20]
criterion = ["entropy", "gini", "error"]
param_grid = [{"max_depth":param_range, "criterion":criterion, "max_leaf_nodes":leaf}]
gs_tree = GridSearchCV(estimator=tree, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=-1)
 gs_tree = gs_tree.fit(X_train, y_train)
 print(gs_tree.best_score_.round(3))
print(gs_tree.best_params_ )
train['date_block_num'] += 1
train['date_block_num']<load_from_csv>
forest = RandomForestClassifier(n_estimators=100, random_state=10)
 param_range = [5, 10, 15]
criterion = ["entropy", "gini", "error"]
param_grid = [{"n_estimators":param_range, "criterion":criterion}]
gs_forest = GridSearchCV(estimator=forest, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=-1)
 gs_forest = gs_forest.fit(X_train, y_train)
 print(gs_forest.best_score_.round(3))
print(gs_forest.best_params_ )
shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv');
print(shops.shape)
 shops.head()<load_from_csv>
xgbc = xgb.XGBClassifier(random_state=10)
 max_depth = [10, 15, 20, 25]
min_samples_leaf = [1,3,5]
min_samples_split = [1,2,4]
param_grid = [{"max_depth":max_depth,
"min_samples_leaf":min_samples_leaf, "min_samples_split":min_samples_split}]
gs_xgb = GridSearchCV(estimator=xgbc, param_grid=param_grid, scoring="accuracy", cv=5, n_jobs=-1)
 gs_xgb = gs_xgb.fit(X_train, y_train)
 print(gs_xgb.best_score_)
 print(gs_xgb.best_params_ )
items= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv');
print(items.shape)
 items.head()<load_from_csv>
lgbm = lgb.LGBMClassifier()
max_depth = [3, 5, 10]
min_samples_leaf = [1,3,5,7]
min_samples_split = [2, 4, 6, 8]
param_grid = [{"max_depth":max_depth,
"min_samples_leaf":min_samples_leaf, "min_samples_split":min_samples_split}]
gs_lgbm = GridSearchCV(estimator=lgbm, param_grid=param_grid, scoring="accuracy", cv=5, n_jobs=-1)
 gs_lgbm = gs_lgbm.fit(X_train, y_train)
 print(gs_lgbm.best_score_.round(3))
print(gs_lgbm.best_params_ )
cats= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv');
print(cats.shape)
 cats.head()<load_from_csv>
print("-"*50)
 y_pred = gs_lr.best_estimator_.predict(X_test_std)
 print("Logistic Regression Result")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_svm.best_estimator_.predict(X_test_std)
 print("Support vector machine, SVM")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_knn.best_estimator_.predict(X_test_std)
 print("KNeithborsClassfier")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_tree.best_estimator_.predict(X_test)
 print("Decision tree")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_forest.best_estimator_.predict(X_test)
 print("Random Forest")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_xgb.best_estimator_.predict(X_test)
 print("XGB")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_lgbm.best_estimator_.predict(X_test)
 print("LGBM")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50 )
sample_submission= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv');
print(sample_submission.shape)
 sample_submission.head()<filter>
test = df_test[['Pclass', 'SibSp', 'Parch', 'Sex_cate', 'Age_band', 'Fare_band', 'Embarked_flg']]
y_pred_test = gs_tree.best_estimator_.predict(test)
 submit = pd.DataFrame({"PassengerId":df_test["PassengerId"], "Survived":y_pred_test} )
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<filter>
submit.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )
train[train.item_price<0]<feature_engineering>
y_pred_test_lgbm = gs_lgbm.best_estimator_.predict(test)
 y_pred_test_xgb = gs_xgb.best_estimator_.predict(test)
 y_pred_test_en =(y_pred_test*0.3 + y_pred_test_lgbm*0.4 + y_pred_test_xgb*0.3)
 submit_en = pd.DataFrame({"PassengerId":df_test["PassengerId"], "Survived":y_pred_test_en} ).round(0)
 submit_en["Survived"] = [int(i)for i in submit_en["Survived"]]
median = train[(train.date_block_num==4)&(train.shop_id==32)&(train.item_id==2973)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
train[train.item_price<0]<feature_engineering>
submit_en.to_csv('my_submission_en.csv', index=False)
 print("Your submission was successfully saved!" )