Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
py.init_notebook_mode(connected=True)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(3 )
df_sub1["winPlacePerc"] =(df_sub1["winPlacePerc"] + df_sub2["winPlacePerc"] + df_sub3["winPlacePerc"])/ 3
df_sub1 = df_sub1[["Id", "winPlacePerc"]]<load_from_csv>
original_train = train.copy()
full_data = [train, test]
train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset.loc[np.isnan(dataset['Age']), 'Age'] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Master": 2, "Mrs": 3, "Miss": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] ;
df_test = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv" )<merge>
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )
df_sub1 = df_sub1.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id",how="left" )<merge>
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
df_sub1_group = df_sub1.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub1_group["rank"] = df_sub1_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub1_group = df_sub1_group.merge(
 df_sub1_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub1_group["adjusted_perc"] =(df_sub1_group["rank"] - 1)/(df_sub1_group["numGroups"] - 1)
 df_sub1 = df_sub1.merge(df_sub1_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub1["winPlacePerc"] = df_sub1["adjusted_perc"]<feature_engineering>
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).agg(['mean', 'count', 'sum'])
df_sub1.loc[df_sub1.maxPlace == 0, "winPlacePerc"] = 0
df_sub1.loc[df_sub1.maxPlace == 1, "winPlacePerc"] = 1<feature_engineering>
title_and_sex = original_train.copy() [['Name', 'Sex']]
title_and_sex['Title'] = title_and_sex['Name'].apply(get_title)
 title_and_sex['Sex'] = title_and_sex['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_and_sex[['Title', 'Sex']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
subset = df_sub1.loc[df_sub1.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub1.loc[df_sub1.maxPlace > 1, "winPlacePerc"] = new_perc<feature_engineering>
def get_gini_impurity(survived_count, total_count):
survival_prob = survived_count/total_count
not_survival_prob =(1 - survival_prob)
 random_observation_survived_prob = survival_prob
random_observation_not_survived_prob =(1 - random_observation_survived_prob)
 mislabelling_survided_prob = not_survival_prob * random_observation_survived_prob
mislabelling_not_survided_prob = survival_prob * random_observation_not_survived_prob
gini_impurity = mislabelling_survided_prob + mislabelling_not_survided_prob
return gini_impurity
df_sub1.loc[(df_sub1.maxPlace > 1)&(df_sub1.numGroups == 1), "winPlacePerc"] = 0
assert df_sub1["winPlacePerc"].isnull().sum() == 0<feature_engineering>
gini_impurity_starting_node = get_gini_impurity(342, 891)
 gini_impurity_starting_node
df_sub1["winPlacePerc"] = df_sub1["winPlacePerc"]<save_to_csv>
gini_impurity_men = get_gini_impurity(109, 577)
 gini_impurity_men
df_sub1[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False )<load_from_csv>
gini_impurity_women = get_gini_impurity(233, 314)
 gini_impurity_women
print('train')
 train_df = import_data('.. /input/train.csv')
 print('test')
 test_df = import_data('.. /input/test.csv' )<groupby>
men_weight = 577/891
women_weight = 314/891
weighted_gini_impurity_sex_split =(gini_impurity_men * men_weight)+(gini_impurity_women * women_weight)
 sex_gini_decrease = weighted_gini_impurity_sex_split - gini_impurity_starting_node
sex_gini_decrease
def featureEngineering(df):
df_size = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_mean = df.groupby(['matchId','groupId'] ).mean().reset_index()
df_max = df.groupby(['matchId','groupId'] ).max().reset_index()
df_min = df.groupby(['matchId','groupId'] ).min().reset_index()
df_match_mean = df.groupby(['matchId'] ).mean().reset_index()
df_train_max_PG = df.groupby(['matchId','groupId'])['kills'].count().reset_index().groupby('matchId')['kills'].max().reset_index()
df_train_max_PG.columns = ['matchId','max_players_in_group']
df = pd.merge(df, df_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 df = pd.merge(df, df_max, suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 df = pd.merge(df, df_min, suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 df = pd.merge(df, df_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 df = pd.merge(df, df_size, how='left', on=['matchId', 'groupId'])
 df = pd.merge(df, df_train_max_PG, how='left', on=['matchId'])
 return df
train_df = featureEngineering(train_df)
 test_df = featureEngineering(test_df )<filter>
gini_impurity_title_1 = get_gini_impurity(81, 517)
 gini_impurity_title_1
print('Old size: %d' % len(train_df))
train_df = train_df[(train_df.kills <= 35)&(train_df.assists <= 13)&(train_df.boosts <= 16)&(train_df.damageDealt <= 3500)
 &(train_df.DBNOs <= 35)&(train_df.headshotKills <= 22)&(train_df.killStreaks <= 10)&(train_df.assists <= 13)
 &(train_df.longestKill <= 1000)&(train_df.revives <= 20)&(train_df.rideDistance <= 28000)&(train_df.roadKills <= 15)
 &(train_df.swimDistance <= 3500)&(train_df.teamKills <= 10)&(train_df.vehicleDestroys <= 3)&(train_df.walkDistance <= 13000)
 &(train_df.weaponsAcquired <= 60)]
print('New size: %d' % len(train_df))
def headshot_precent(data):
data['headPerc']=data['headshotKills']/data['kills']
data['roadPerc']=data['roadKills']/data['kills']
data['totalDistance']=data['rideDistance']+data['swimDistance']+data['walkDistance']
data['rideDistancePerc']=data['rideDistance']/data['totalDistance']
data['swimDistancePerc']=data['swimDistance']/data['totalDistance']
data['walkDistancePerc']=data['walkDistance']/data['totalDistance']
data.fillna(0,axis=1,inplace=True)
 return data.head()
headshot_precent(train_df)
 headshot_precent(test_df )<drop_column>
gini_impurity_title_others = get_gini_impurity(261, 374)
 gini_impurity_title_others
features_not2use = ['Id', 'groupId', 'matchId','winPlacePerc_mean','winPlacePerc_max','winPlacePerc_min','winPlacePerc_match_mean']
for df in [train_df]:
df.drop(features_not2use, axis=1, inplace=True)
 features_not2use = ['Id', 'groupId', 'matchId']
for df in [test_df]:
df.drop(features_not2use, axis=1, inplace=True)
 X_train = train_df.sample(frac=0.8)
 X_val = train_df.loc[~train_df.index.isin(X_train.index)] 
<prepare_x_and_y>
title_1_weight = 517/891
title_others_weight = 374/891
weighted_gini_impurity_title_split =(gini_impurity_title_1 * title_1_weight)+(gini_impurity_title_others * title_others_weight)
 title_gini_decrease = weighted_gini_impurity_title_split - gini_impurity_starting_node
title_gini_decrease
y_train = X_train['winPlacePerc']
X_train.drop('winPlacePerc', axis=1, inplace=True)
 y_val = X_val['winPlacePerc']
X_val.drop('winPlacePerc', axis=1, inplace=True )<train_model>
cv = KFold(n_splits=10)
 accuracies = list()
max_attributes = len(list(test))
depth_range = range(1, max_attributes + 1)
 for depth in depth_range:
fold_accuracy = []
tree_model = tree.DecisionTreeClassifier(max_depth = depth)
 ")
 for train_fold, valid_fold in cv.split(train):
f_train = train.loc[train_fold]
f_valid = train.loc[valid_fold]
model = tree_model.fit(X = f_train.drop(['Survived'], axis=1),
y = f_train["Survived"])
 valid_acc = model.score(X = f_valid.drop(['Survived'], axis=1),
y = f_valid["Survived"])
 fold_accuracy.append(valid_acc)
 avg = sum(fold_accuracy)/len(fold_accuracy)
 accuracies.append(avg)
 ")
 ")
 df = pd.DataFrame({"Max Depth": depth_range, "Average Accuracy": accuracies})
 df = df[["Max Depth", "Average Accuracy"]]
print(df.to_string(index=False))
start_time = time.time()
model = LGBMRegressor(iterations=250, learning_rate=0.05, loss_function='MAE',eval_metric='MAE', depth = -1,
use_best_model=True, od_type="Iter", od_wait=10, thread_count=128, random_seed = 123, num_leaves= 144,n_estimators= 800,
bagging_fraction= 0.8, bagging_freq= 5, feature_fraction= 0.9, objective= 'regression_l2')
 model.fit(X_train, y_train, eval_set=(X_val, y_val))
end_time = time.time()
print('The training time = {}'.format(end_time - start_time))<predict_on_test>
y_train = train['Survived']
x_train = train.drop(['Survived'], axis=1 ).values
x_test = test.values
decision_tree = tree.DecisionTreeClassifier(max_depth = 3)
 decision_tree.fit(x_train, y_train)
 y_pred = decision_tree.predict(x_test)
 submission = pd.DataFrame({
"PassengerId": PassengerId,
"Survived": y_pred
})
 submission.to_csv('submission.csv', index=False)
 with open("tree1.dot", 'w')as f:
f = tree.export_graphviz(decision_tree,
out_file=f,
max_depth = 3,
impurity = True,
feature_names = list(train.drop(['Survived'], axis=1)) ,
class_names = ['Died', 'Survived'],
rounded = True,
filled= True)
 check_call(['dot','-Tpng','tree1.dot','-o','tree1.png'])
 img = Image.open("tree1.png")
 draw = ImageDraw.Draw(img)
 font = ImageFont.truetype('/usr/share/fonts/truetype/liberation/LiberationSerif-Bold.ttf', 26)
 draw.text(( 10, 0),
'"Title <= 1.5" corresponds to "Mr." title', 
(0,0,255),
font=font)
 img.save('sample-out.png')
 PImage("sample-out.png")
pred = model.predict(test_df )<save_to_csv>
acc_decision_tree = round(decision_tree.score(x_train, y_train)* 100, 2)
 acc_decision_tree