Kaggle competition: Titanic - Machine Learning from Disaster
count_2011 = df['year'][df['year'] == 2011].count()
count_2012 = df['year'][df['year'] == 2012].count()
print('2011: {}'.format(count_2011))
print('2012: {}'.format(count_2012))<statistical_test>
_, _, r_value, _, _ = stats.linregress(df['count'], df['temp'])
 r_square = r_value ** 2
r_square.round(2 )<statistical_test>
_, _, r_value, _, _ = stats.linregress(df['count'], df['atemp'])
 r_square = r_value ** 2
r_square.round(2 )<statistical_test>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.describe(include="all" )
_, _, r_value, _, _ = stats.linregress(df['temp'], df['atemp'])
 r_square = r_value ** 2
r_square.round(2 )<drop_column>
hasCabin =(train["Cabin"].notnull().astype('int'))
sns.set_style('whitegrid')
 sns.countplot(x='Survived', hue=hasCabin, data=train)
 print("Percentage by has Cabin: ", train["Survived"][hasCabin == 1].value_counts(normalize=True)[1] * 100)
 print("Percentage of hasn't Cabin: ", train["Survived"][hasCabin == 0].value_counts(normalize=True)[1] * 100 )
df = df.drop(labels='atemp', axis=1 )<drop_column>
train["Age"].fillna(train["Age"].median() , inplace = True)
 test["Age"].fillna(test["Age"].median() , inplace = True)
 train["Fare"].fillna(train["Fare"].median() , inplace = True)
 test["Fare"].fillna(test["Fare"].median() , inplace = True)
 train["Embarked"].fillna(( train["Embarked"].append(test["Embarked"])).mode() , inplace = True)
 test["Embarked"].fillna(( train["Embarked"].append(test["Embarked"])).mode() , inplace = True)
 drop_column = ["PassengerId","Cabin", "Ticket"]
train.drop(drop_column, axis=1, inplace = True)
 print(train.isnull().sum())
 print("-"*10)
 print(test.isnull().sum() )
df = df.drop(labels='count_log', axis=1 )<drop_column>
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1
train["IsAlone"] = 1
test["IsAlone"] = 1
train["IsAlone"].loc[train["FamilySize"] > 1] = 0
test["IsAlone"].loc[test["FamilySize"] > 1] = 0
train["Title"] = train["Name"].str.split(",", expand = True)[1].str.split(".", expand = True)[0]
test["Title"] = test["Name"].str.split(",", expand = True)[1].str.split(".", expand = True)[0]
train["FareBin"] = pd.qcut(train["Fare"], 4)
 test["FareBin"] = pd.qcut(test["Fare"], 4)
 train["AgeBin"] = pd.cut(train["Age"].astype(int), 5)
 test["AgeBin"] = pd.cut(test["Age"].astype(int), 5 )
df = df.drop(labels='count_boxcox', axis=1 )<categorify>
stat_min = 10
title_names =(train["Title"].value_counts() < stat_min)
 title_names_t =(test["Title"].value_counts() < stat_min)
 train["Title"] = train["Title"].apply(lambda x: "Misc" if title_names.loc[x] == True else x)
 test["Title"] = test["Title"].apply(lambda x: "Misc" if title_names_t.loc[x] == True else x)
 print(train["Title"].value_counts())
 print("-"*10)
 train.info()
test.info()
train.sample(10 )
df = pd.get_dummies(df, columns=['weather'])
 df.head()<drop_column>
label = preprocessing.LabelEncoder()
train["Sex_Code"] = label.fit_transform(train["Sex"])
 test["Sex_Code"] = label.fit_transform(test["Sex"])
 train["Embarked_Code"] = label.fit_transform(train["Embarked"].astype(str))
test["Embarked_Code"] = label.fit_transform(test["Embarked"].astype(str))
train["Title_Code"] = label.fit_transform(train["Title"])
 test["Title_Code"] = label.fit_transform(test["Title"])
 train["AgeBin_Code"] = label.fit_transform(train["AgeBin"])
 test["AgeBin_Code"] = label.fit_transform(test["AgeBin"])
 train["FareBin_Code"] = label.fit_transform(train["FareBin"])
 test["FareBin_Code"] = label.fit_transform(test["FareBin"] )
df = df.drop(labels='weather_4', axis=1)
 df.head()<feature_engineering>
Target = ["Survived"]
train_x = ["Sex", "Pclass", "Embarked", "Title", "SibSp", "Parch", "Age", "Fare", "FamilySize", "IsAlone"]
train_x_calc = ["Sex_Code", "Pclass", "Embarked_Code", "Title_Code", "SibSp", "Parch", "Age", "Fare"]
train_x_bin = ["Sex_Code", "Pclass", "Embarked_Code", "Title_Code", "FamilySize", "AgeBin_Code", "FareBin_Code"]
df['temp_weath_1'] = df['temp'] * df['weather_1']
df['temp_weath_2'] = df['temp'] * df['weather_2']
df['temp_weath_3'] = df['temp'] * df['weather_3']<data_type_conversions>
train_dummy = pd.get_dummies(train[train_x])
 train_x_dummy = train_dummy.columns.tolist()
train_xy_dummy = Target + train_x_dummy
print("Dummy X Y: ", train_xy_dummy, '
')
 train_dummy.head()
df['temp_weath_1'] = df['temp_weath_1'].astype(int)
 df['temp_weath_2'] = df['temp_weath_2'].astype(int)
 df['temp_weath_3'] = df['temp_weath_3'].astype(int )<prepare_x_and_y>
test_dummy = pd.get_dummies(test[train_x])
 test_x_dummy = train_dummy.columns.tolist()
test_xy_dummy = Target + test_x_dummy
print("Dummy X Y: ", test_xy_dummy, '
')
 test_dummy.head()
X = df.loc[:, df.columns != 'count']
y = np.log(df['count'] )<import_modules>
x_train, x_test, y_train, y_test = train_test_split(train[train_x_calc], train[Target], test_size = 0.22, random_state = 0)
 x_train_bin, x_test_bin, y_train_bin, y_test_bin = train_test_split(train[train_x_bin], train[Target], test_size = 0.22, random_state = 0)
 x_train_dummy, x_test_dummy, y_train_dummy, y_test_dummy = \
train_test_split(train_dummy[train_x_dummy], train[Target], test_size = 0.22, random_state = 0)
 print("Original Train Shape: {}".format(train.shape))
print("Train Shape: {}".format(np.asarray(x_train ).shape))
print("Test Shape: {}".format(np.asarray(x_test ).shape))
from sklearn.model_selection import train_test_split<import_modules>
from sklearn.model_selection import train_test_split<split>
gaussian = GaussianNB()
gaussian.fit(x_train_dummy, y_train_dummy)
 y_hat_train = gaussian.predict(x_train_dummy)
 y_hat_test = gaussian.predict(x_test_dummy)
 acc_gaussian = gaussian.score(x_test_dummy, y_test_dummy)
 print('Gaussian Naive Bayes train accuracy score: ', gaussian.score(x_train_dummy, y_train_dummy))
print('Gaussian Naive Bayes test accuracy score: ', acc_gaussian)
 print('Gaussian Naive Bayes train accuracy score: ', round(accuracy_score(y_hat_train, y_train_dummy)* 100, 2))
print('Gaussian Naive Bayes test accuracy score: ', round(accuracy_score(y_hat_test, y_test_dummy)* 100, 2))
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2 )<import_modules>
logreg = LogisticRegression()
logreg.fit(x_train_dummy, y_train_dummy)
 y_hat_train = logreg.predict(x_train_dummy)
 y_hat_test = logreg.predict(x_test_dummy)
 acc_logreg = logreg.score(x_test_dummy, y_test_dummy)
 print('Logistic Regression train accuracy score: ', logreg.score(x_train_dummy, y_train_dummy))
print('Logistic Regression test accuracy score: ', acc_logreg )
from sklearn.linear_model import LinearRegression, Lasso, ElasticNet, Ridge
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler, MaxAbsScaler, MinMaxScaler, Normalizer, minmax_scale, QuantileTransformer, RobustScaler, PolynomialFeatures
from sklearn.model_selection import KFold, cross_val_score
from xgboost import XGBRegressor<load_from_csv>
svc = SVC()
svc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = svc.predict(x_train_dummy)
 y_hat_test = svc.predict(x_test_dummy)
 cv_score = cross_val_score(svc, x_train_dummy, y_train_dummy, cv=10)
 plt.plot(cv_score)
 acc_svc = svc.score(x_test_dummy, y_test_dummy)
 print('Support Vector Machines train accuracy score: ', svc.score(x_train_dummy, y_train_dummy))
print('Support Vector Machines test accuracy score: ', acc_svc)
 print('Support Vector Machines cross validation train accuracy score: ', cv_score.mean() )
df_test = pd.read_csv(".. /input/test.csv" )<data_type_conversions>
linear_svc = LinearSVC()
linear_svc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = linear_svc.predict(x_train_dummy)
 y_hat_test = linear_svc.predict(x_test_dummy)
 acc_linear_svc = linear_svc.score(x_test_dummy, y_test_dummy)
 print('Linear SVC train accuracy score: ', linear_svc.score(x_train_dummy, y_train_dummy))
print('Linear SVC test accuracy score: ', acc_linear_svc )
df_test['datetime'] = pd.to_datetime(df_test['datetime'] )<feature_engineering>
perceptron = Perceptron()
perceptron.fit(x_train_dummy, y_train_dummy)
 y_hat_train = perceptron.predict(x_train_dummy)
 y_hat_test = perceptron.predict(x_test_dummy)
 acc_perceptron = perceptron.score(x_test_dummy, y_test_dummy)
 print('Perceptron train accuracy score: ', perceptron.score(x_train_dummy, y_train_dummy))
print('Perceptron test accuracy score: ', acc_perceptron )
df_test['dow'] = df_test['datetime'].dt.dayofweek<feature_engineering>
decisiontree = DecisionTreeClassifier()
decisiontree.fit(x_train_dummy, y_train_dummy)
 y_hat_train = decisiontree.predict(x_train_dummy)
 y_hat_test = decisiontree.predict(x_test_dummy)
 acc_decisiontree = decisiontree.score(x_test_dummy, y_test_dummy)
 print('Decision Tree train accuracy score: ', decisiontree.score(x_train_dummy, y_train_dummy))
print('Decision Tree test accuracy score: ', acc_decisiontree )
df_test['month'] = df_test['datetime'].dt.month<feature_engineering>
randomforest = RandomForestClassifier()
randomforest.fit(x_train_dummy, y_train_dummy)
 y_hat_train = randomforest.predict(x_train_dummy)
 y_hat_test = randomforest.predict(x_test_dummy)
 acc_randomforest = randomforest.score(x_test_dummy, y_test_dummy)
 print('Random Forest train accuracy score: ', randomforest.score(x_train_dummy, y_train_dummy))
print('Random Forest test accuracy score: ', acc_randomforest )
df_test['week'] = df_test['datetime'].dt.week<feature_engineering>
knn = KNeighborsClassifier()
knn.fit(x_train_dummy, y_train_dummy)
 y_hat_train = knn.predict(x_train_dummy)
 y_hat_test = knn.predict(x_test_dummy)
 acc_knn = knn.score(x_test_dummy, y_test_dummy)
 print('KNN train accuracy score: ', knn.score(x_train_dummy, y_train_dummy))
print('KNN test accuracy score: ', acc_knn )
df_test['hour'] = df_test['datetime'].dt.hour<feature_engineering>
sgd = SGDClassifier()
sgd.fit(x_train_dummy, y_train_dummy)
 y_hat_train = sgd.predict(x_train_dummy)
 y_hat_test = sgd.predict(x_test_dummy)
 acc_sgd = sgd.score(x_test_dummy, y_test_dummy)
 print('SGD train accuracy score: ', sgd.score(x_train_dummy, y_train_dummy))
print('SGD test accuracy score: ', acc_sgd )
df_test['year'] = df_test['datetime'].dt.year<feature_engineering>
gbk = GradientBoostingClassifier()
gbk.fit(x_train_dummy, y_train_dummy)
 y_hat_train = gbk.predict(x_train_dummy)
 y_hat_test = gbk.predict(x_test_dummy)
 acc_gbk = gbk.score(x_test_dummy, y_test_dummy)
 print('SGD train accuracy score: ', gbk.score(x_train_dummy, y_train_dummy))
print('SGD test accuracy score: ', acc_gbk )
df_test['day'] = df_test['datetime'].dt.day<rename_columns>
ada = AdaBoostClassifier()
ada.fit(x_train_dummy, y_train_dummy)
 y_hat_train = ada.predict(x_train_dummy)
 y_hat_test = ada.predict(x_test_dummy)
 acc_ada = ada.score(x_test_dummy, y_test_dummy)
 print('Ada Boost train accuracy score: ', ada.score(x_train_dummy, y_train_dummy))
print('Ada Boost accuracy score: ', acc_ada )
df_test = df_test.set_index(df_test['datetime'] )<drop_column>
models = pd.DataFrame({
"Model": ["SVM", "KNN", "Logistic Regression", "Random Forest",
"Naive Bayes", "Perceptron", "Linear SVC",
"Decision Tree", "SGD", "GBK", "Ada Boost"],
"Score": [acc_svc, acc_knn, acc_logreg, acc_randomforest,
acc_gaussian, acc_perceptron, acc_linear_svc,
acc_decisiontree, acc_sgd, acc_gbk, acc_ada]
})
 models.sort_values(by="Score", ascending=False )
df_test = df_test.drop(labels='datetime', axis=1 )<drop_column>
eclf_hard = VotingClassifier(estimators = [("DecisionTree", decisiontree), 
("GBK", gbk), 
("Logistic", logreg), 
("Linear SVC", linear_svc), 
("Ada Boost", ada), 
("KNN", knn), 
('SVM', svc)], voting="hard", weights=[6, 2, 1, 0.8, 3.5, 5, 4])
 eclf_hard.fit(x_train_dummy, y_train_dummy)
 print('Emsemble Voting train accuracy score: ', eclf_hard.score(x_train_dummy, y_train_dummy))
print('Emsemble Voting accuracy score: ', eclf_hard.score(x_test_dummy, y_test_dummy))
df_test = df_test.drop(labels='atemp', axis=1 )<categorify>
mlp = MLPClassifier(max_iter=50)
 parameter_space = {
'hidden_layer_sizes': [(50,50,50),(50,100,50),(100,)],
'activation': ['tanh', 'relu'],
'solver': ['lbfgs', 'adam'],
'alpha': [0.0001, 0.05],
'learning_rate': ['constant','adaptive'],
}
gridsearch = GridSearchCV(mlp, parameter_space, n_jobs=-1, cv=3)
 gridsearch.fit(x_train_dummy, y_train_dummy)
 print("Best parameters found: ", gridsearch.best_params_)
 means = gridsearch.cv_results_['mean_test_score']
stds = gridsearch.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, gridsearch.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r" %(mean, std * 2, params))
print('Results on the train set:')
 print(classification_report(y_train, gridsearch.predict(x_train_dummy)))
 print('Results on the test set:')
 print(classification_report(y_test, gridsearch.predict(x_test_dummy)))
df_test = pd.get_dummies(df_test, columns=['weather'] )<drop_column>
xgbc = XGBClassifier()
xgbc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = xgbc.predict(x_train_dummy)
 y_hat_test = xgbc.predict(x_test_dummy)
 print('Xgboost train accuracy score: ', xgbc.score(x_train_dummy, y_train_dummy))
print('Xgboost test accuracy score: ', xgbc.score(x_test_dummy, y_test_dummy))
print('Results on the test set: ')
 print(classification_report(y_test_dummy, y_hat_test))
df_test = df_test.drop(labels='weather_4', axis=1 )<feature_engineering>
predictions = gbk.predict(test_dummy)
 output = pd.DataFrame({ 'PassengerId' : test["PassengerId"], "Survived": predictions })
 output.to_csv("gbk_submission.csv", index=False )
df_test['temp_weath_1'] = df_test['temp'] * df_test['weather_1']
df_test['temp_weath_2'] = df_test['temp'] * df_test['weather_2']
df_test['temp_weath_3'] = df_test['temp'] * df_test['weather_3']<data_type_conversions>
predictions = randomforest.predict(test_dummy)
 output = pd.DataFrame({ 'PassengerId' : test["PassengerId"], "Survived": predictions })
 output.to_csv("rf_submission.csv", index=False )
df_test['temp_weath_1'] = df_test['temp_weath_1'].astype(int)
 df_test['temp_weath_2'] = df_test['temp_weath_2'].astype(int)
 df_test['temp_weath_3'] = df_test['temp_weath_3'].astype(int )<choose_model_class>
predictions = eclf_hard.predict(test_dummy)
 output = pd.DataFrame({ 'PassengerId' : test["PassengerId"], "Survived": predictions })
 output.to_csv("ensmble_submission.csv", index=False )
standardscaler = StandardScaler()
model = XGBRegressor(colsample_bytree=0.7, learning_rate=0.05, max_depth=7, min_child_weight=4, subsample=0.7, random_state=42 )<train_model>
predictions = gridsearch.predict(test_dummy)
 output = pd.DataFrame({ 'PassengerId' : test["PassengerId"], "Survived": predictions })
 output.to_csv("gd_mlp_submission.csv", index=False )
model.fit(X_train, y_train )<train_model>
model.fit(X_train, y_train )<predict_on_test>