Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import randint as randint
from scipy.stats import uniform
import lightgbm as lgbm
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV
from sklearn.metrics import accuracy_score
lb_sub_category_1 = LabelBinarizer(sparse_output=True)
 X_main_sub_cat_1_train = lb_sub_category_1.fit_transform(X_train['sub_category_1'])
 X_main_sub_cat_1_cv = lb_sub_category_1.transform(X_cv['sub_category_1'] )<categorify>
def load_data(data_path, **kwargs):
return pd.read_csv(data_path, header=0, delimiter=kwargs['delimiter'])
 def clean_titanic_dataset(base_data_org, new_data_org, create_dummy=False):

 base_data = base_data_org.copy(deep=True)
 new_data = new_data_org.copy(deep=True)
 if(type(base_data)!= type(pd.DataFrame())) or(type(new_data)!= type(pd.DataFrame())) :
print("Input data should be pandas dataframe.")
 return None
if('PassengerId' not in base_data)or('PassengerId' not in new_data):
print("PassengerId attribute is missing.")
 return None
if(base_data.PassengerId.isnull().sum() != 0)or(new_data.PassengerId.isnull().sum() != 0):
print("PassengerId cannot have missing value.")
 return None
for item in new_data.loc[new_data.Pclass.isnull() ,:].iterrows() :
fare = item[1]["Fare"]
PassengerId = item[1]["PassengerId"]
if(fare != fare):
new_data.loc[new_data["PassengerId"]==PassengerId,"Pclass"] = base_data.Pclass.mode()
new_data.loc[new_data["PassengerId"]==PassengerId,"Fare"] = base_data.Fare.median()
else:
pclass_impute =(base_data.groupby('Pclass')['Fare'].median() - fare ).abs().idxmin()
new_data.loc[new_data["PassengerId"]==PassengerId,"Pclass"] = pclass_impute
new_data["Pclass"] = new_data["Pclass"].astype(int)
 new_data.loc[new_data.SibSp != new_data.SibSp, "SibSp"] = base_data.SibSp.mode() [0]
new_data.loc[new_data.Parch != new_data.Parch, "Parch"] = base_data.Parch.mode() [0]
new_data["Family_size"] = new_data["SibSp"] + new_data["Parch"] +1
new_data["Family_type"] = "Alone"
new_data.loc[new_data["Family_size"] > 1, "Family_type"] = "Small"
new_data.loc[new_data["Family_size"] > 4, "Family_type"] = "Big"
new_data.loc[new_data.Fare != new_data.Fare, "Fare"] = base_data.Fare.median()
new_data['Adj_fare'] = new_data['Fare'] / new_data['Family_size']
new_data["Cabin_assigned"] = 1
new_data.loc[new_data.Cabin != new_data.Cabin,"Cabin_assigned"] = 0
new_data.loc[new_data.Embarked != new_data.Embarked,"Embarked"] = base_data.Embarked.mode() [0]
for my_data in [base_data,new_data]:
my_data["Title"] = my_data.Name.str.extract('\w+,\s+([a-zA-z\s]+)\s*\.. *')
 my_data["Last_Name"] = my_data.Name.str.extract('(\w+),\s+[a-zA-z\s]+\s*\.. *')
 title_set = set(base_data.Title.unique())
 title_set.update(['Other','Special'])
 mrs = ["Mrs","Mme"]
miss = ["Mlle","Miss","Ns"]
other = ["Capt","Col","Major","Dr","Rev","Mlle","Mme","Ms","Ms",]
special = ["Lady", "Don", "Dona", "Sir", "the Countess", "Jonkheer"]
for my_data in [base_data,new_data]:
my_data["Title"].replace(mrs,"Mrs", inplace=True)
 my_data["Title"].replace(miss,"Miss", inplace=True)
 my_data["Title"].replace(other,"Other", inplace=True)
 my_data["Title"].replace(special,"Special", inplace=True)
 Titles = ["Mr","Mrs","Miss","Master","Other","Special"]
new_data.loc[new_data["Title"].apply(lambda x: x not in title_set),"Title"] = "Other"
new_data["Sex_numerical"] = new_data["Sex"].map({"male": 1, "female": 0} ).astype(int)
 new_data["Age_missing"] = 0
new_data.loc[new_data.Age.isnull() ,"Age_missing"] = 1
for title in Titles:
for pclass in range(1,4):
try:
agg_df = base_data.groupby(['Title','Pclass'])['Age'].median()
age_to_impute = np.asscalar(agg_df.loc[[(title, pclass)]].values)
 new_data.loc[(new_data['Age'].isnull())&(new_data['Title'] == title)&(new_data['Pclass'] == pclass), 'Age'] = age_to_impute
except:
agg_df = base_data.groupby(['Title'])['Age'].median()
age_to_impute = np.asscalar(agg_df.loc[[(title)]].values)
 new_data.loc[(new_data['Age'].isnull())&(new_data['Title'] == title), 'Age'] = age_to_impute
new_data["Child"] = 0
new_data.loc[new_data["Age"]<18,"Child"] = 1
new_data['Fsize_age'] = new_data['Family_size'] + new_data['Age']/70
new_data['Adj_fare'] = new_data['Fare'] / new_data['Family_size']
if create_dummy:
new_data = pd.concat([new_data, pd.get_dummies(new_data["Embarked"], prefix='Embarked')], axis=1)
 new_data = pd.concat([new_data, pd.get_dummies(new_data["Title"], prefix='Title')], axis=1)
 new_data = pd.concat([new_data, pd.get_dummies(new_data["Pclass"], prefix='Pclass')], axis=1)
 new_data = pd.concat([new_data, pd.get_dummies(new_data["Family_type"], prefix='Ftype')], axis=1)
 return new_data
def plot_feature_importance(model,X_train,model_name,num_features):
fig, axes = plt.subplots(nrows = 1, ncols = 1, sharex="all", figsize=(10,3))
indices = np.argsort(model.feature_importances_)[::-1][:num_features]
g = sns.barplot(y = X_train.columns[indices][:num_features],
x = model.feature_importances_[indices][:num_features] , orient='h')
 g.set_xlabel("Relative importance",fontsize=12)
 g.set_ylabel("Features",fontsize=12)
 g.tick_params(labelsize=9)
 g.set_title(model_name + " feature importance");
def evaluate_model_with_test_data(test_set,features,model):
Y_test = test_set["Survived"]
X_test = test_set[features]
return accuracy_score(Y_test, model.predict(X_test))
def retrain_with_whole_data(data,model,features):
Y_train = data["Survived"]
X_train = data[features]
return model.fit(X_train,Y_train)
 def predict_test_data(model, data, features):
X_test_prediction = data[features]
return pd.Series(model.predict(X_test_prediction ).astype(int), name="Survived")
lb_sub_category_2 = LabelBinarizer(sparse_output=True)
 X_main_sub_cat_2_train = lb_sub_category_2.fit_transform(X_train['sub_category_2'])
 X_main_sub_cat_2_cv = lb_sub_category_2.transform(X_cv['sub_category_2'] )<categorify>
train_data = load_data('.. /input/titanic/train.csv', delimiter = ',')
 test_data = load_data('.. /input/titanic/test.csv', delimiter = ',' )
X_dummies_train = csr_matrix(
 pd.get_dummies(X_train[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_train_1 = csr_matrix(
 pd.get_dummies(X_train[list({'item_condition_id', 'shipping'})],
sparse=True ).values )<categorify>
tr_data_c = clean_titanic_dataset(train_data,train_data, create_dummy=True)
 ts_data_c = clean_titanic_dataset(train_data,test_data, create_dummy=True)
 IDtest = ts_data_c["PassengerId"]
X_dummies_cv = csr_matrix(
 pd.get_dummies(X_cv[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_cv_1 = csr_matrix(
 pd.get_dummies(X_cv[list({'item_condition_id', 'shipping'})],
sparse=True ).values )<concatenate>
tr_data_c.set_index("PassengerId",inplace=True)
 ts_data_c.set_index("PassengerId",inplace=True )
sparse_merge_train = hstack(( X_name_train_wb , X_description_train_wb, X_brand_train, X_main_cat_train,
X_main_sub_cat_1_train, X_main_sub_cat_2_train,X_dummies_train,X_dummies_train_1)).tocsr()<concatenate>
train_data_cleaned, test_data_cleaned = df.loc[tr_data_c.index], df.loc[ts_data_c.index]
sparse_merge_cv = hstack(( X_name_cv_wb,X_description_cv_wb,X_brand_cv,X_main_cat_cv,
X_main_sub_cat_1_cv,X_main_sub_cat_2_cv,X_dummies_cv,X_dummies_cv_1)).tocsr()<drop_column>
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=914)
 for train_index, test_index in split.split(train_data_cleaned, train_data_cleaned[["Sex"]]):
train_set = train_data_cleaned.iloc[train_index]
test_set = train_data_cleaned.iloc[test_index]
X_dummies_train = None
X_description_train_wb = None
X_name_train_wb = None
X_dummies_cv = None
X_description_cv_wb = None
X_name_cv_wb = None
del(X_dummies_train, X_description_train_wb,X_name_train_wb)
 del(X_dummies_cv, X_description_cv_wb,X_name_cv_wb)
 gc.collect()<load_pretrained>
lgbm_grid_search = False
lgbm_feature_names = [
'Sex_numerical'
, 'Family_survival_rate'
, 'Is_single_traveler'
]
Y_train = train_set["Survived"]
X_train = train_set[lgbm_feature_names]
if lgbm_grid_search:
fit_params = {"early_stopping_rounds" : 100,
"eval_metric" : 'auc',
"eval_set" : [(X_train,Y_train)],
'eval_names': ['valid'],
'verbose': 0,
'categorical_feature': 'auto'}
param_test = {'learning_rate' : [0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.2, 0.3, 0.4],
'n_estimators' : [100, 200, 300, 400, 500, 600, 800, 1000, 1500, 2000],
'num_leaves': randint(6, 200),
'min_child_samples': randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': uniform(loc = 0.2, scale = 0.8),
'max_depth': [-1, 1, 2, 3, 4, 5, 6, 7,8,9,10],
'colsample_bytree': uniform(loc = 0.4, scale = 0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 0.01, 0.1, 0.3, 1, 5, 10, 20, 50, 100]}
lgbm_clf = lgbm.LGBMClassifier(random_state = 914, silent = True, metric = 'None', n_jobs = 16)
 grid_search = RandomizedSearchCV(
 estimator = lgbm_clf, param_distributions = param_test,
n_iter = 2000,
scoring = 'accuracy',
cv = 5,
refit = True,
random_state = 914,
verbose = True)
 grid_search.fit(X_train, Y_train, **fit_params)
 opt_parameters = grid_search.best_params_
print(grid_search.best_params_)
 else:
opt_parameters = {
'colsample_bytree': 0.951896848025216,
'learning_rate': 0.2,
'max_depth': 10,
'min_child_samples': 102,
'min_child_weight': 0.01,
'n_estimators': 400,
'num_leaves': 102,
'reg_alpha': 2,
'reg_lambda': 0.5,
'subsample': 0.4194694182848429}
lgbm_best = lgbm.LGBMClassifier(**opt_parameters)
 lgbm_model_best_ = lgbm_best.fit(X_train,Y_train )
pickle.dump(lb_brand_name, open("lb_brand_name.pickle", "wb"),protocol=4)
 pickle.dump(lb_main_category, open("lb_main_category.pickle", "wb"),protocol=4)
 pickle.dump(lb_sub_category_1, open("lb_sub_category_1.pickle", "wb"),protocol=4)
 pickle.dump(lb_sub_category_2, open("lb_sub_category_2.pickle", "wb"),protocol=4 )<drop_column>
evaluate_model_with_test_data(test_set,lgbm_feature_names,lgbm_model_best_ )
lb_brand_name = None
lb_main_category = None
lb_sub_category_1 = None
lb_sub_category_2 = None
del(lb_brand_name,lb_main_category,lb_sub_category_1,lb_sub_category_2)
 gc.collect()<train_model>
lgbm_on_training = retrain_with_whole_data(train_data_cleaned,lgbm_model_best_,lgbm_feature_names)
 test_Survived_lgbm = predict_test_data(lgbm_on_training, test_data_cleaned,lgbm_feature_names )
<predict_on_test><EOS> 
results = pd.concat([IDtest,test_Survived_lgbm],axis=1)
 results.to_csv("submission.csv",index=False )