Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_train_metric>
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
 %matplotlib inline
cv=cross_val_score(model,train,y_train,scoring=log_scorer,cv=5)
 cv<save_to_csv>
data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv' )
sub = pd.read_csv(".. /input/sample_submission.csv")
 sub['shot_made_flag'] = target_y
sub.to_csv("submission.csv", index=False )<set_options>
print(data.isnull().sum())
 print(test_data.isnull().sum() )
sns.set_style('darkgrid')
 sns.set_palette('bone')
 pd.options.display.float_format = '{:,.3f}'.format<load_from_csv>
data.groupby(['Sex', 'Survived'])['Survived'].count()
df = pd.read_csv(".. /input/data.csv")
 df.shape<count_unique_values>
data['Initial'] = 0
for i in data:
data['Initial'] = data.Name.str.extract('([A-Za-z]+)\.')
 test_data['Initial'] = 0
for i in test_data:
test_data['Initial'] = test_data.Name.str.extract('([A-Za-z]+)\.' )
df.game_id.nunique() , df.game_date.nunique()<count_unique_values>
data['Initial'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don'],['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr'],inplace=True)
 test_data['Initial'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don','Dona'],['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr','Mr'],inplace=True )
df.game_event_id.nunique()<count_values>
data.groupby('Initial')['Age'].mean()
df.action_type.value_counts() [:10]<filter>
data.loc[(data.Age.isnull())&(data.Initial=='Mr'),'Age']=33
data.loc[(data.Age.isnull())&(data.Initial=='Mrs'),'Age']=36
data.loc[(data.Age.isnull())&(data.Initial=='Master'),'Age']=5
data.loc[(data.Age.isnull())&(data.Initial=='Miss'),'Age']=22
data.loc[(data.Age.isnull())&(data.Initial=='Other'),'Age']=46
test_data.loc[(test_data.Age.isnull())&(test_data.Initial=='Mr'),'Age']=33
test_data.loc[(test_data.Age.isnull())&(test_data.Initial=='Mrs'),'Age']=36
test_data.loc[(test_data.Age.isnull())&(test_data.Initial=='Master'),'Age']=5
test_data.loc[(test_data.Age.isnull())&(test_data.Initial=='Miss'),'Age']=22
test_data.loc[(test_data.Age.isnull())&(test_data.Initial=='Other'),'Age']=46
_ = df[(df["minutes_remaining"] == 0)&(df["seconds_remaining"] <= 10)]
_.mean() ["shot_made_flag"], _.count() ["shot_made_flag"]<feature_engineering>
data.Age.isnull().any()
test_data.Age.isnull().any()
df["game_year"] = df["game_date"].str[0:4].astype(int)
 df["game_month"] = df["game_date"].str[5:7].astype(int)
 df['action_first_words'] = df["action_type"].str.split(' ' ).str[0]
df['action_last_words'] = df["action_type"].str.split(' ' ).str[-2]
df['season_start_year'] = df.season.str.split('-' ).str[0].astype(int)
 df["remaining"] = df["minutes_remaining"] * 60 + df["seconds_remaining"]
df["hurry_shot"] =(( df["minutes_remaining"] == 0)&(df["seconds_remaining"] < 10)).astype(int)
 df["home_game"] = df["matchup"].apply(lambda x: 1 if(x.find('@')< 0)else 0)
 df['distance_bin'] = pd.cut(df.shot_distance, bins=10, labels=range(10)).astype('int')
 df["angle"] = df.apply(lambda row: 90 if row["loc_y"]==0 else m.degrees(m.atan(row["loc_x"]/abs(row["loc_y"]))),axis=1)
 df["angle_bin"] = pd.cut(df.angle, 7, labels=range(7)).astype(int )<drop_column>
data['Embarked'].fillna('S', inplace=True)
 test_data['Embarked'].fillna('S', inplace=True )
df.drop(["team_id", "team_name", "game_date", "game_event_id", "matchup"], axis=1, inplace=True )<count_missing_values>
data.Embarked.isnull().any()
test_data.Embarked.isnull().any()
nullcount = df.isnull().sum()
nullcount[nullcount > 0]<concatenate>
pd.crosstab(data.SibSp, data.Pclass ).style.background_gradient(cmap='summer_r' )
_ = pd.concat([df.game_id, df.period, df.shot_made_flag, df.game_id.shift(1), df.period.shift(1), df.shot_made_flag.shift(1)], axis=1)
 _.columns = ["game_id", "period", "shot_made_flag", "pre_game_id", "pre_period", "pre_shot_made_flag"]
_.dropna()
_ = _[(_["game_id"] == _["pre_game_id"])&(_["period"] == _["pre_period"])]
_.groupby(["pre_shot_made_flag"] ).mean() ["shot_made_flag"]<prepare_output>
pd.crosstab(data.Parch, data.Pclass ).style.background_gradient(cmap='summer_r' )
df_enc = df.copy()<categorify>
data['Age_band'] = 0
data.loc[data['Age']<=16, 'Age_band'] = 0
data.loc[(data['Age']>16)&(data['Age']<=32), 'Age_band'] = 1
data.loc[(data['Age']>32)&(data['Age']<=48), 'Age_band'] = 2
data.loc[(data['Age']>48)&(data['Age']<=64), 'Age_band'] = 3
data.loc[data['Age']>64, 'Age_band'] = 4
data.head(2)
 test_data['Age_band'] = 0
test_data.loc[test_data['Age']<=16, 'Age_band'] = 0
test_data.loc[(test_data['Age']>16)&(test_data['Age']<=32), 'Age_band'] = 1
test_data.loc[(test_data['Age']>32)&(test_data['Age']<=48), 'Age_band'] = 2
test_data.loc[(test_data['Age']>48)&(test_data['Age']<=64), 'Age_band'] = 3
test_data.loc[test_data['Age']>64, 'Age_band'] = 4
test_data.head(2 )
for i, t in df_enc.dtypes.iteritems() :
if t == object:
le = LabelEncoder()
le.fit(df_enc[i].astype(str))
df_enc[i] = le.transform(df_enc[i].astype(str))<filter>
data['Fare_cat']=0
data.loc[data['Fare']<=7.91,'Fare_cat']=0
data.loc[(data['Fare']>7.91)&(data['Fare']<=14.454),'Fare_cat']=1
data.loc[(data['Fare']>14.454)&(data['Fare']<=31),'Fare_cat']=2
data.loc[(data['Fare']>31)&(data['Fare']<=513),'Fare_cat']=3
test_data['Fare_cat']=0
test_data.loc[test_data['Fare']<=7.91,'Fare_cat']=0
test_data.loc[(test_data['Fare']>7.91)&(test_data['Fare']<=14.454),'Fare_cat']=1
test_data.loc[(test_data['Fare']>14.454)&(test_data['Fare']<=31),'Fare_cat']=2
test_data.loc[(test_data['Fare']>31)&(test_data['Fare']<=513),'Fare_cat']=3
train = df[~df.shot_made_flag.isnull() ]<sort_values>
data['Sex'].replace(['male','female'],[0,1],inplace=True)
 data['Embarked'].replace(['S','C','Q'],[0,1,2],inplace=True)
 data['Initial'].replace(['Mr','Mrs','Miss','Master','Other'],[0,1,2,3,4],inplace=True)
 test_data['Sex'].replace(['male','female'],[0,1],inplace=True)
 test_data['Embarked'].replace(['S','C','Q'],[0,1,2],inplace=True)
 test_data['Initial'].replace(['Mr','Mrs','Miss','Master','Other'],[0,1,2,3,4],inplace=True )
def shot_mean(group_col):
return train.groupby([group_col] ).mean() ["shot_made_flag"]
def sorted_shot_mean(group_col):
return train.groupby([group_col] ).mean() ["shot_made_flag"].sort_values(ascending=False )<prepare_x_and_y>
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix
X_train = df_enc[~df_enc.shot_made_flag.isnull() ]
X_game_id = X_train.pop('game_id')
 Y_train = X_train['shot_made_flag']
X_train = X_train.drop(['shot_id','shot_made_flag'], axis=1)
 X_test = df_enc[df_enc.shot_made_flag.isnull() ].drop(['game_id','shot_id','shot_made_flag'], axis=1 )<split>
train, test = train_test_split(data, test_size=0.3, random_state=0, stratify=data['Survived'])
 train_X = train[train.columns[1:]]
train_Y = train[train.columns[:1]]
test_X = test[test.columns[1:]]
test_Y = test[test.columns[:1]]
X = data[data.columns[1:]]
Y = data['Survived']
params={'learning_rate': 0.03,
'objective':'binary',
'metric':'binary_logloss',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 1,
'feature_fraction': 0.8
}
folds = GroupKFold(n_splits=10)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train, groups=X_game_id)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train.iloc[trn_]
val_x, val_y = X_train.iloc[val_], Y_train.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=3000)
 reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=100, verbose=500)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 sub_preds += reg.predict(X_test, num_iteration=reg.best_iteration_)/ folds.n_splits
pred = sub_preds<save_to_csv>
model = svm.SVC(kernel='rbf', C=1, gamma=0.1)
 model.fit(train_X, train_Y)
 prediction1 = model.predict(test_X)
 print('Accuracy for rbf SVM is ', metrics.accuracy_score(prediction1, test_Y))
submission = pd.DataFrame({
"shot_id": df[df.shot_made_flag.isnull() ]["shot_id"],
"shot_made_flag": pred
})
 submission.to_csv("submission.csv", index=False )<import_modules>
model=svm.SVC(kernel='linear',C=0.1,gamma=0.1)
 model.fit(train_X,train_Y)
 prediction2=model.predict(test_X)
 print('Accuracy for linear SVM is',metrics.accuracy_score(prediction2,test_Y))
import numpy as np
import pandas as pd
import xgboost as xgb<load_from_csv>
model = LogisticRegression()
model.fit(train_X, train_Y)
 prediction3 = model.predict(test_X)
 print('The accuracy of the Logistic Regression is', metrics.accuracy_score(prediction3, test_Y))
data = pd.read_csv('.. /input/data.csv')
 data.set_index('shot_id', inplace=True )<prepare_x_and_y>
model = DecisionTreeClassifier()
model.fit(train_X, train_Y)
 prediction4 = model.predict(test_X)
 print('The accuracy of the Decision Tree is', metrics.accuracy_score(prediction4, test_Y))
unknown_mask = data['shot_made_flag'].isnull()
data_cl = data.copy()
target = data_cl['shot_made_flag'].copy()<drop_column>
model = KNeighborsClassifier()
model.fit(train_X, train_Y)
 prediction5 = model.predict(test_X)
 print('The accuracy of the KNN is', metrics.accuracy_score(prediction5, test_Y))
data_cl.drop('team_id', inplace=True, axis=1)
 data_cl.drop('lat', inplace=True, axis=1)
 data_cl.drop('lon', inplace=True, axis=1)
 data_cl.drop('game_id', inplace=True, axis=1)
 data_cl.drop('game_event_id', inplace=True, axis=1)
 data_cl.drop('team_name', inplace=True, axis=1)
 data_cl.drop('shot_made_flag', inplace=True, axis=1 )<feature_engineering>
a_index = list(range(1,11))
a = pd.Series()
x = [0,1,2,3,4,5,6,7,8,9,10]
for i in list(range(1,11)) :
model = KNeighborsClassifier(n_neighbors=i)
 model.fit(train_X, train_Y)
 prediction = model.predict(test_X)
 a = a.append(pd.Series(metrics.accuracy_score(prediction, test_Y)))
 plt.plot(a_index, a)
 plt.xticks(x)
 plt.show()
print('Accuracies for different values of n are : ', a.values, 'with the max value as ', a.values.max() )
data_cl['seconds_from_period_end'] = 60 * data_cl['minutes_remaining'] + data_cl['seconds_remaining']
data_cl['last_5_sec_in_period'] = data_cl['seconds_from_period_end'] < 5
data_cl['seconds_from_period_start'] = 60*(11-data_cl['minutes_remaining'])+(60-data_cl['seconds_remaining'])
 data_cl['seconds_from_game_start'] =(data_cl['period'] <= 4 ).astype(int)*(data_cl['period']-1)*12*60 +(data_cl['period'] > 4 ).astype(int)*(( data_cl['period']-4)*5*60 + 3*12*60)+ data_cl['seconds_from_period_start']
data_cl.drop('minutes_remaining', axis=1, inplace=True)
 data_cl.drop('seconds_remaining', axis=1, inplace=True )<data_type_conversions>
model = GaussianNB()
model.fit(train_X, train_Y)
 prediction6 = model.predict(test_X)
 print('The accuracy of the NaiveBayes is ', metrics.accuracy_score(prediction6, test_Y))
data_cl['home_play'] = data_cl['matchup'].str.contains('vs' ).astype('int')
 data_cl.drop('matchup', axis=1, inplace=True )<feature_engineering>
model=RandomForestClassifier(n_estimators=100)
 model.fit(train_X,train_Y)
 prediction7=model.predict(test_X)
 print('The accuracy of the Random Forests is',metrics.accuracy_score(prediction7,test_Y))
data_cl['game_date'] = pd.to_datetime(data_cl['game_date'])
 data_cl['game_year'] = data_cl['game_date'].dt.year
data_cl['game_month'] = data_cl['game_date'].dt.month
data_cl['dayOfWeek'] = data_cl['game_date'].dt.dayofweek
data_cl['dayOfYear'] = data_cl['game_date'].dt.dayofyear
data_cl.drop('game_date', axis=1, inplace=True )<count_values>
plt.subplots(figsize=(12,6))
box = pd.DataFrame(accuracy, index=[classifiers])
 box.T.boxplot()
rare_action_types = data_cl['action_type'].value_counts().sort_values().index.values[:20]
data_cl.loc[data_cl['action_type'].isin(rare_action_types), 'action_type'] = 'Other'<categorify>
C = [0.05, 0.1, 0.2, 0.3, 0.25, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
gamma = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
kernel = ['rbf', 'linear']
hyper = {'kernel': kernel, 'C': C, 'gamma': gamma}
gd = GridSearchCV(estimator=svm.SVC() , param_grid=hyper, verbose=True)
 gd.fit(X,Y)
 print(gd.best_score_)
 print(gd.best_estimator_ )
categorial_cols = [
'action_type', 'combined_shot_type', 'period', 'season', 'shot_type',
'shot_zone_area', 'shot_zone_basic', 'shot_zone_range', 'game_year',
'game_month', 'opponent']
for cc in categorial_cols:
dummies = pd.get_dummies(data_cl[cc])
 dummies = dummies.add_prefix("{}_".format(cc))
data_cl.drop(cc, axis=1, inplace=True)
 data_cl = data_cl.join(dummies )<prepare_x_and_y>
n_estimators = range(100, 1000, 100)
 hyper = {'n_estimators': n_estimators}
gd = GridSearchCV(estimator = RandomForestClassifier(random_state=0), param_grid=hyper, verbose=True)
 gd.fit(X, Y)
 print(gd.best_score_)
 print(gd.best_estimator_ )
data_submit = data_cl[unknown_mask]
X = data_cl[~unknown_mask]
Y = target[~unknown_mask]<prepare_x_and_y>
ensemble_lin_rbf = VotingClassifier(estimators=[ 
('KNN', KNeighborsClassifier(n_neighbors=10)) , 
('RBF', svm.SVC(probability=True, kernel='rbf', C=0.5, gamma=0.1)) , 
('RFor', RandomForestClassifier(n_estimators=500, random_state=0)) , 
('LR',LogisticRegression(C=0.05)) , 
('DT',DecisionTreeClassifier(random_state=0)) , 
('NB', GaussianNB()), 
('svm', svm.SVC(kernel='linear', probability=True))
], voting='soft' ).fit(train_X, train_Y)
 print('The accracy for ensembled model is : ', ensemble_lin_rbf.score(test_X, test_Y))
cross = cross_val_score(ensemble_lin_rbf, X, Y, cv = 10, scoring = 'accuracy')
 print('The cross validated score is', cross.mean() )
d_train = xgb.DMatrix(X, label=Y)
 dtest = xgb.DMatrix(data_submit )<init_hyperparams>
model = BaggingClassifier(base_estimator=KNeighborsClassifier(n_neighbors=3), random_state=0, n_estimators=700)
 model.fit(train_X, train_Y)
 prediction = model.predict(test_X)
 print('The accuracy for bagged KNN is : ', metrics.accuracy_score(prediction, test_Y))
result = cross_val_score(model, X, Y, cv = 10, scoring='accuracy')
 print('The cross validated score for bagged KNN is : ', result.mean() )
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['max_depth'] = 7
params['silent'] = 1
params['colsample_bytree'] = 0.7
params['eta'] = 0.004
params['max_delta_step'] = 1
params['min_child_weight'] = 3<compute_test_metric>
ada = AdaBoostClassifier(n_estimators=200, random_state=0, learning_rate=0.1)
 result = cross_val_score(ada, X, Y, cv = 10, scoring='accuracy')
 print('The cross validated score for AdaBoost is : ', result.mean() )
<train_model>
grad = GradientBoostingClassifier(n_estimators=500, random_state=0, learning_rate=0.1)
 result = cross_val_score(grad,X, Y, cv = 10, scoring='accuracy')
 print('The cross validated score for Gradient Boosting is : ', result.mean() )
clf = xgb.train(params, d_train, num_boost_round=961 )<save_to_csv>
xgboost = xg.XGBClassifier(n_estimators=900, learning_rate=0.1)
 result = cross_val_score(xgboost, X, Y, cv=10, scoring='accuracy')
 print('The cross validated score for XGBoost is : ', result.mean() )
preds = clf.predict(dtest)
 submission = pd.DataFrame()
submission["shot_id"] = data_submit.index
submission["shot_made_flag"]= preds
submission.to_csv("sub_xgb.csv",index=False )<load_from_csv>
n_estimators = list(range(100, 1100, 100))
learn_rate = [0.05, 0.1, 0.2, 0.3, 0.25, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
hyper = {'n_estimators' : n_estimators, 'learning_rate' : learn_rate}
gd = GridSearchCV(estimator=AdaBoostClassifier() , param_grid=hyper, verbose=True)
 gd.fit(X,Y)
 print(gd.best_score_)
 print(gd.best_estimator_ )
<prepare_x_and_y><EOS> 
model=AdaBoostClassifier(n_estimators=200,learning_rate=0.05,random_state=0)
 model.fit(X,Y)
 prediction = test_data
prediction['Survived'] = model.predict(test_data.drop(['PassengerId'], axis=1))
pd.DataFrame(prediction[['PassengerId', 'Survived']] ).to_csv('submission.csv', index=False )