Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<init_hyperparams>
%matplotlib inline
warnings.filterwarnings('ignore' )
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':100,
"num_leaves" : 25, "learning_rate" : 0.05, "bagging_fraction" : 0.9, "feature_fraction":0.7,
"bagging_seed" : 0, "num_threads" : 4
}<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.describe(include="all" )
for n_fold,(train_idx, vali_idx)in enumerate(folds.split(X_train, y_train)) :
X_train_fold, y_train_fold = X_train.iloc[train_idx], y_train[train_idx]
X_vali, y_vali = X_train.iloc[vali_idx], y_train[vali_idx]
train_data = lgb.Dataset(data=X_train_fold, label=y_train_fold)
 valid_data = lgb.Dataset(data=X_vali, label=y_vali)
 lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000)
 pred_fold = lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)
 pred_fold[pred_fold>1] = 1
pred_fold[pred_fold<0] = 0
pred += pred_fold/ folds.n_splits
vali_pred[vali_idx] = lgb_model.predict(X_vali, num_iteration=lgb_model.best_iteration)
 vali_pred[vali_pred>1] = 1
vali_pred[vali_pred<0] = 0
print('Fold %2d MAE : %.6f' %(n_fold + 1, mean_absolute_error(y_vali, vali_pred[vali_idx])))
 df_fold_importance = pd.DataFrame()
df_fold_importance = df_fold_importance.assign(feature= lst_features)
 df_fold_importance = df_fold_importance.assign(importance = lgb_model.feature_importance())
 df_fold_importance = df_fold_importance.assign(fold = n_fold + 1)
 df_feature_importance = pd.concat([df_feature_importance, df_fold_importance])
 gc.collect() 
<compute_test_metric>
print(pd.isnull(train ).sum() )
print('Full mae score %.6f' % mean_absolute_error(y_train, vali_pred))<save_model>
train = train.drop(['Cabin'], axis = 1)
 test = test.drop(['Cabin'], axis = 1 )
lgb_model.save_model('lgb_model.txt' )<merge>
train = train.drop(['Ticket'], axis = 1)
 test = test.drop(['Ticket'], axis = 1 )
pred = np.clip(pred, a_min=0, a_max=1)
 df_pred = X_test_index.assign(winPlacePerc=pred)
 result = pd.merge(df_test_ids, df_pred, how='left', on=['matchId', 'groupId'] )<save_to_csv>
train = train.fillna({"Embarked": "S"} )
submission = result[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<create_dataframe>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace(['Countess', 'Lady', 'Sir'], 'Royal')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
def get_null_observations(dataframe, column):
return dataframe[pd.isnull(dataframe[column])]
def delete_null_observations(dataframe, column):
fixed_df = dataframe.drop(get_null_observations(dataframe,column ).index)
 return fixed_df
def get_missing_data_table(dataframe):
total = dataframe.isnull().sum()
percentage = dataframe.isnull().sum() / dataframe.isnull().count()
missing_data = pd.concat([total, percentage], axis='columns', keys=['TOTAL','PERCENTAGE'])
 return missing_data.sort_index(ascending=True)
 df = pd.read_csv('.. /input/train_V2.csv')
 df.head()<count_missing_values>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Royal": 5, "Rare": 6}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train.head()
get_missing_data_table(df )<create_dataframe>
mr_age = train[train["Title"] == 1]["AgeGroup"].mode()
miss_age = train[train["Title"] == 2]["AgeGroup"].mode()
mrs_age = train[train["Title"] == 3]["AgeGroup"].mode()
master_age = train[train["Title"] == 4]["AgeGroup"].mode()
royal_age = train[train["Title"] == 5]["AgeGroup"].mode()
rare_age = train[train["Title"] == 6]["AgeGroup"].mode()
age_title_mapping = {1: "Young Adult", 2: "Student", 3: "Adult", 4: "Baby", 5: "Adult", 6: "Adult"}
for x in range(len(train["AgeGroup"])) :
if train["AgeGroup"][x] == "Unknown":
train["AgeGroup"][x] = age_title_mapping[train["Title"][x]]
for x in range(len(test["AgeGroup"])) :
if test["AgeGroup"][x] == "Unknown":
test["AgeGroup"][x] = age_title_mapping[test["Title"][x]]
df = delete_null_observations(dataframe=df, column='winPlacePerc')
 get_missing_data_table(df )<categorify>
age_mapping = {'Baby': 1, 'Child': 2, 'Teenager': 3, 'Student': 4, 'Young Adult': 5, 'Adult': 6, 'Senior': 7}
train['AgeGroup'] = train['AgeGroup'].map(age_mapping)
 test['AgeGroup'] = test['AgeGroup'].map(age_mapping)
 train.head()
train = train.drop(['Age'], axis = 1)
 test = test.drop(['Age'], axis = 1 )
df_team_dict =(df.groupby('groupId', as_index = True)
.agg({'Id':'count', 'kills':'sum'})
.rename(columns={'Id':'teamSize', 'kills':'teamKills'})).to_dict()
teamKills = []
teamSize = []
for teamId in df['groupId']:
teamKills.append(df_team_dict['teamKills'][teamId])
 teamSize.append(df_team_dict['teamSize'][teamId])
 df['teamKills'] = teamKills
df['teamSize'] = teamSize
df.head()<groupby>
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )
df_team =(df.groupby('groupId', as_index = False)
.agg({'Id':'count', 'matchId':lambda x: x.unique() [0], 'kills':'sum'})
.rename(columns={'Id':'teamSize', 'kills':'teamKills'})).reset_index()
df_match =(df_team.groupby('matchId', as_index = True)
.agg({'teamSize':'sum', 'teamKills':'sum'})
.rename(columns={'teamSize':'matchSize', 'teamKills':'matchKills'})).to_dict()
matchSize = []
matchKills = []
for matchId in df['matchId']:
matchSize.append(df_match['matchSize'][matchId])
 matchKills.append(df_match['matchKills'][matchId])
 df['matchSize'] = matchSize
df['matchKills'] = matchKills
df.head()<drop_column>
sex_mapping = {"male": 0, "female": 1}
train['Sex'] = train['Sex'].map(sex_mapping)
 test['Sex'] = test['Sex'].map(sex_mapping)
 train.head()
df.drop(['Id'], axis='columns', inplace=True)
 df.drop(['groupId'], axis='columns', inplace=True)
 df.drop(['matchId'], axis='columns', inplace=True)
 df.head()<define_variables>
embarked_mapping = {"S": 1, "C": 2, "Q": 3}
train['Embarked'] = train['Embarked'].map(embarked_mapping)
 test['Embarked'] = test['Embarked'].map(embarked_mapping)
 train.head()
previous_record_size = df.shape[0]
h_spread = df['matchDuration'].quantile (.75)- df['matchDuration'].quantile (.25)
 limit = df['matchDuration'].quantile (.25)- 2 * h_spread
df.drop(df[df['matchDuration'] < limit].index, inplace=True)
 new_record_size = df.shape[0]
print('Total records deleted: {}({:.7%} of previous record size)'.format(previous_record_size - new_record_size, 1 - new_record_size / previous_record_size))<drop_column>
predictors = train.drop(['Survived', 'PassengerId'], axis=1)
 target = train["Survived"]
x_train, x_val, y_train, y_val = train_test_split(predictors, target, test_size = 0.22, random_state = 0 )
previous_record_size = df.shape[0]
df.drop(df.query('rideDistance == 0 and roadKills > 0' ).index, inplace=True)
 new_record_size = df.shape[0]
print('Total records deleted: {}({:.7%} of previous record size)'.format(previous_record_size - new_record_size, 1 - new_record_size / previous_record_size))<categorify>
gaussian = GaussianNB()
gaussian.fit(x_train, y_train)
 y_pred = gaussian.predict(x_val)
 acc_gaussian = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gaussian )
encoder = preprocessing.LabelEncoder()
df['matchType'] = encoder.fit_transform(df['matchType'])
 df.head()<prepare_x_and_y>
logreg = LogisticRegression()
logreg.fit(x_train, y_train)
 y_pred = logreg.predict(x_val)
 acc_logreg = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_logreg )
y = df['winPlacePerc'].values
X = df.drop(['winPlacePerc'], axis='columns' ).values<split>
svc = SVC()
svc.fit(x_train, y_train)
 y_pred = svc.predict(x_val)
 acc_svc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_svc )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
 lgb_train = lgb.Dataset(X_train, y_train, categorical_feature=[12])
 lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
 params = {
"objective" : "regression",
"metric" : "mae",
"n_estimators":15000,
"early_stopping_rounds":100,
"num_leaves" : 31,
"learning_rate" : 0.05,
"bagging_fraction" : 0.9,
"bagging_seed" : 0,
"num_threads" : 4,
"colsample_bytree" : 0.7
}
model = lgb.train(params,
lgb_train,
num_boost_round=20,
valid_sets=lgb_eval,
early_stopping_rounds=5,
verbose_eval=1000 )<load_from_csv>
linear_svc = LinearSVC()
linear_svc.fit(x_train, y_train)
 y_pred = linear_svc.predict(x_val)
 acc_linear_svc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_linear_svc )
df_test = pd.read_csv('.. /input/test_V2.csv')
 df_test['matchType'] = encoder.transform(df_test['matchType'])
 df_test_team_dict =(df_test.groupby('groupId', as_index = True)
.agg({'Id':'count', 'kills':'sum'})
.rename(columns={'Id':'teamSize', 'kills':'teamKills'})).to_dict()
teamKills_test = []
teamSize_test = []
for teamId in df_test['groupId']:
teamKills_test.append(df_test_team_dict['teamKills'][teamId])
 teamSize_test.append(df_test_team_dict['teamSize'][teamId])
 df_test['teamKills'] = teamKills_test
df_test['teamSize'] = teamSize_test
df_team_test =(df_test.groupby('groupId', as_index = False)
.agg({'Id':'count', 'matchId':lambda x: x.unique() [0], 'kills':'sum'})
.rename(columns={'Id':'teamSize', 'kills':'teamKills'})).reset_index()
df_match_test =(df_team_test.groupby('matchId', as_index = True)
.agg({'teamSize':'sum', 'teamKills':'sum'})
.rename(columns={'teamSize':'matchSize', 'teamKills':'matchKills'})).to_dict()
matchSize_test = []
matchKills_test = []
for matchId in df_test['matchId']:
matchSize_test.append(df_match_test['matchSize'][matchId])
 matchKills_test.append(df_match_test['matchKills'][matchId])
 df_test['matchSize'] = matchSize_test
df_test['matchKills'] = matchKills_test
X_testdata = df_test.drop(['Id','groupId','matchId'], axis='columns' ).values
df_test['winPlacePerc'] = model.predict(X_testdata, num_iteration=model.best_iteration)
 submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False)
 print('Done!' )<load_from_csv>
perceptron = Perceptron()
perceptron.fit(x_train, y_train)
 y_pred = perceptron.predict(x_val)
 acc_perceptron = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_perceptron )
def reduce_mem_usage_func(df):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
def get_sampled_data(filename, sample_size):
n = sum(1 for line in open(filename)) - 1
skip = sorted(random.sample(range(1,n+1),n-sample_size))
df = pd.read_csv(filename, skiprows=skip)
 return df
def read_fast(filename, sample=True, sample_size=200000, reduce_mem_usage=True):
start_time = time.time()
df = get_sampled_data(filename, sample_size)if sample else pd.read_csv(filename)
 new_df = reduce_mem_usage_func(df)if reduce_mem_usage else df
elapsed_time = int(time.time() - start_time)
 print('Time to get data frame: {:02d}:{:02d}:{:02d}'.format(
 elapsed_time // 3600, 
(elapsed_time % 3600 // 60),
elapsed_time % 60))
return new_df 
<set_options>
decisiontree = DecisionTreeClassifier()
decisiontree.fit(x_train, y_train)
 y_pred = decisiontree.predict(x_val)
 acc_decisiontree = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_decisiontree )
warnings.filterwarnings('ignore')
 % matplotlib inline 
<load_from_csv>
randomforest = RandomForestClassifier()
randomforest.fit(x_train, y_train)
 y_pred = randomforest.predict(x_val)
 acc_randomforest = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_randomforest )
data = read_fast(".. /input/train_V2.csv", sample = False)
 data.head(10 )<load_from_csv>
knn = KNeighborsClassifier()
knn.fit(x_train, y_train)
 y_pred = knn.predict(x_val)
 acc_knn = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_knn )
test_data = read_fast(".. /input/test_V2.csv", sample = False )<filter>
sgd = SGDClassifier()
sgd.fit(x_train, y_train)
 y_pred = sgd.predict(x_val)
 acc_sgd = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_sgd )
data[data['winPlacePerc'].isnull() ]<feature_engineering>
gbk = GradientBoostingClassifier()
gbk.fit(x_train, y_train)
 y_pred = gbk.predict(x_val)
 acc_gbk = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gbk )
data['headshotrate'] = data['headshotKills']/data['kills']
data['healthitems'] = data['heals'] + data['boosts']
data['totalDistance'] = data['rideDistance'] + data["walkDistance"] + data["swimDistance"]
data['killPlace_over_maxPlace'] = data['killPlace'] / data['maxPlace']
data['killsPerWalkDistance'] = data['kills'] /(data['walkDistance'] + 1)
 data['allKills'] = data['headshotKills']+data['kills']+data['roadKills']+data['teamKills']+data['assists']<feature_engineering>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron', 'Linear SVC',
'Decision Tree', 'Stochastic Gradient Descent', 'Gradient Boosting Classifier'],
'Score': [acc_svc, acc_knn, acc_logreg,
acc_randomforest, acc_gaussian, acc_perceptron,acc_linear_svc, acc_decisiontree,
acc_sgd, acc_gbk]})
 models.sort_values(by='Score', ascending=False )
test_data['headshotrate'] =test_data['headshotKills']/ test_data['kills']
test_data['healthitems'] = test_data['heals'] + test_data['boosts']
test_data['totalDistance'] = test_data['rideDistance'] + test_data["walkDistance"] + test_data["swimDistance"]
test_data['killPlace_over_maxPlace'] = test_data['killPlace'] / test_data['maxPlace']
test_data['killsPerWalkDistance'] = test_data['kills'] /(test_data['walkDistance'] +1)
 test_data['allKills'] = test_data['headshotKills']+test_data['kills']+test_data['roadKills']+test_data['teamKills']+test_data['assists']<feature_engineering>
ids = test['PassengerId']
predictions = gbk.predict(test.drop('PassengerId', axis=1))
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )