Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
preds = []
ids = []<predict_on_test>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
for batch in chunker(test_files, batch_size):
X = [preprocess_input(cv2.imread(x)) for x in batch]
ids_batch = [get_id_from_file_path(x)for x in batch]
X = np.array(X)
 preds_batch =(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
preds += preds_batch
ids += ids_batch<save_to_csv>
all_data = pd.concat([train,test],sort=False,ignore_index=True)
 all_data.info()
df = pd.DataFrame({'id':ids, 'label':preds})
 df.to_csv("baseline_nasnet.csv", index=False)
 df.head()<set_options>
train.groupby('Pclass' ).Survived.agg(['mean','size'] )
pd.options.mode.chained_assignment = None
warnings.filterwarnings("ignore", category=DeprecationWarning)
 %matplotlib inline<load_from_csv>
train.groupby('Sex')['Survived'].mean()
dailyData = pd.read_csv(".. /input/train.csv" )<feature_engineering>
train['family_size'] = train.SibSp+train.Parch+1
all_data['family_size'] = all_data.SibSp+all_data.Parch+1
dailyData["date"] = dailyData.datetime.apply(lambda x : x.split() [0])
 dailyData["hour"] = dailyData.datetime.apply(lambda x : x.split() [1].split(":")[0])
 dailyData["weekday"] = dailyData.date.apply(lambda dateString : calendar.day_name[datetime.strptime(dateString,"%Y-%m-%d" ).weekday() ])
 dailyData["month"] = dailyData.date.apply(lambda dateString : calendar.month_name[datetime.strptime(dateString,"%Y-%m-%d" ).month])
 dailyData["season"] = dailyData.season.map({1: "Spring", 2 : "Summer", 3 : "Fall", 4 :"Winter" })
 dailyData["weather"] = dailyData.weather.map({1: " Clear + Few clouds + Partly cloudy + Partly cloudy",\
2 : " Mist + Cloudy, Mist + Broken clouds, Mist + Few clouds, Mist ", \
3 : " Light Snow, Light Rain + Thunderstorm + Scattered clouds, Light Rain + Scattered clouds", \
4 :" Heavy Rain + Ice Pallets + Thunderstorm + Mist, Snow + Fog " } )<data_type_conversions>
train.groupby('family_size' ).Survived.mean()
categoryVariableList = ["hour","weekday","month","season","weather","holiday","workingday"]
for var in categoryVariableList:
dailyData[var] = dailyData[var].astype("category" )<drop_column>
all_data.Fare[all_data.Fare.isnull() ] = train.Fare.median()
dailyData = dailyData.drop(["datetime"],axis=1 )<filter>
cabin_first = train.Cabin.map(lambda x:str(x)[0] )
dailyDataWithoutOutliers = dailyData[np.abs(dailyData["count"]-dailyData["count"].mean())<=(3*dailyData["count"].std())]<load_from_csv>
train.Survived.groupby(cabin_first ).agg(['mean','size'] )
dataTrain = pd.read_csv(".. /input/train.csv")
 dataTest = pd.read_csv(".. /input/test.csv" )<concatenate>
train.groupby('Embarked' ).Survived.agg(['mean','size'] )
data = dataTrain.append(dataTest)
 data.reset_index(inplace=True)
 data.drop('index',inplace=True,axis=1 )<feature_engineering>
all_data.Embarked.fillna(train.Embarked.mode() [0],inplace=True )
data["date"] = data.datetime.apply(lambda x : x.split() [0])
 data["hour"] = data.datetime.apply(lambda x : x.split() [1].split(":")[0] ).astype("int")
 data["year"] = data.datetime.apply(lambda x : x.split() [0].split("-")[0])
 data["weekday"] = data.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).weekday())
 data["month"] = data.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).month )<choose_model_class>
train.groupby(train.Ticket.map(lambda x: str(x)[0])).Survived.agg(['mean','count'] )
dataWind0 = data[data["windspeed"]==0]
dataWindNot0 = data[data["windspeed"]!=0]
rfModel_wind = RandomForestRegressor()
windColumns = ["season","weather","humidity","month","temp","year","atemp"]
rfModel_wind.fit(dataWindNot0[windColumns], dataWindNot0["windspeed"])
 wind0Values = rfModel_wind.predict(X= dataWind0[windColumns])
 dataWind0["windspeed"] = wind0Values
data = dataWindNot0.append(dataWind0)
 data.reset_index(inplace=True)
 data.drop('index',inplace=True,axis=1 )<define_variables>
ticket_num = train.Ticket.str.extract('^(\d)|^\D.*\s(\d)\d*$' ).fillna(0 ).astype(int ).sum(axis=1 )
categoricalFeatureNames = ["season","holiday","workingday","weather","weekday","month","year","hour"]
numericalFeatureNames = ["temp","humidity","windspeed","atemp"]
dropFeatures = ['casual',"count","datetime","date","registered"]<data_type_conversions>
train.Survived.groupby(ticket_num ).agg(['mean','count'] )
for var in categoricalFeatureNames:
data[var] = data[var].astype("category" )<prepare_x_and_y>
test.groupby(test.Ticket.map(lambda x: str(x)[0])).size()
dataTrain = data[pd.notnull(data['count'])].sort_values(by=["datetime"])
 dataTest = data[~pd.notnull(data['count'])].sort_values(by=["datetime"])
 datetimecol = dataTest["datetime"]
yLabels = dataTrain["count"]
yLablesRegistered = dataTrain["registered"]
yLablesCasual = dataTrain["casual"]<drop_column>
all_data['Ticket_first_letter'] = all_data.Ticket.map(lambda x: str(x)[0])
 all_data['Ticket_first_num'] = all_data.Ticket.str.extract('^(\d)|^\D.*\s(\d)\d*$' ).fillna(0 ).astype(int ).sum(axis=1 )
dataTrain = dataTrain.drop(dropFeatures,axis=1)
 dataTest = dataTest.drop(dropFeatures,axis=1 )<compute_test_metric>
names = all_data.Name.str.extract('^ (.*),(\w+)')
 all_data['family_name'] = names.loc[:,0]
all_data['honorific'] = names.loc[:,1]
def rmsle(y, y_,convertExp=True):
if convertExp:
y = np.exp(y),
y_ = np.exp(y_)
 log1 = np.nan_to_num(np.array([np.log(v + 1)for v in y]))
log2 = np.nan_to_num(np.array([np.log(v + 1)for v in y_]))
calc =(log1 - log2)** 2
return np.sqrt(np.mean(calc))<choose_model_class>
all_data.iloc[:891,:].groupby('honorific' ).Survived.agg(['mean','size'] )
pd.options.mode.chained_assignment = None
warnings.filterwarnings("ignore", category=DeprecationWarning)
 lModel = LinearRegression()
yLabelsLog = np.log1p(yLabels)
 lModel.fit(X = dataTrain,y = yLabelsLog)
 preds = lModel.predict(X= dataTrain)
 print("RMSLE Value For Linear Regression: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))<train_on_grid>
all_data.iloc[891:,:].groupby('honorific' ).size()
ridge_m_ = Ridge()
ridge_params_ = { 'max_iter':[3000],'alpha':[0.1, 1, 2, 3, 4, 10, 30,100,200,300,400,800,900,1000]}
rmsle_scorer = metrics.make_scorer(rmsle, greater_is_better=False)
 grid_ridge_m = GridSearchCV(ridge_m_,
ridge_params_,
scoring = rmsle_scorer,
cv=5)
 yLabelsLog = np.log1p(yLabels)
 grid_ridge_m.fit(dataTrain, yLabelsLog)
 preds = grid_ridge_m.predict(X= dataTrain)
 print(grid_ridge_m.best_params_)
 print("RMSLE Value For Ridge Regression: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))
fig,ax= plt.subplots()
fig.set_size_inches(12,5)
 df = pd.DataFrame(grid_ridge_m.grid_scores_)
 df["alpha"] = df["parameters"].apply(lambda x:x["alpha"])
 df["rmsle"] = df["mean_validation_score"].apply(lambda x:-x)
 sn.pointplot(data=df,x="alpha",y="rmsle",ax=ax )<train_on_grid>
name_map={
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the": "Royalty",
"Dona": "Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty"}
all_data.honorific.replace(name_map,inplace=True )
lasso_m_ = Lasso()
alpha = 1/np.array([0.1, 1, 2, 3, 4, 10, 30,100,200,300,400,800,900,1000])
 lasso_params_ = { 'max_iter':[3000],'alpha':alpha}
grid_lasso_m = GridSearchCV(lasso_m_,lasso_params_,scoring = rmsle_scorer,cv=5)
 yLabelsLog = np.log1p(yLabels)
 grid_lasso_m.fit(dataTrain, yLabelsLog)
 preds = grid_lasso_m.predict(X= dataTrain)
 print(grid_lasso_m.best_params_)
 print("RMSLE Value For Lasso Regression: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))
fig,ax= plt.subplots()
fig.set_size_inches(12,5)
 df = pd.DataFrame(grid_lasso_m.grid_scores_)
 df["alpha"] = df["parameters"].apply(lambda x:x["alpha"])
 df["rmsle"] = df["mean_validation_score"].apply(lambda x:-x)
 sn.pointplot(data=df,x="alpha",y="rmsle",ax=ax )<train_model>
all_data.groupby(['honorific','Sex'] ).Survived.agg(['mean','size'] )
rfModel = RandomForestRegressor(n_estimators=100)
 yLabelsLog = np.log1p(yLabels)
 rfModel.fit(dataTrain,yLabelsLog)
 preds = rfModel.predict(X= dataTrain)
 print("RMSLE Value For Random Forest: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))<predict_on_test>
all_data.groupby(['honorific','Sex'] ).Age.mean()
gbm = GradientBoostingRegressor(n_estimators=4000,alpha=0.01);
yLabelsLog = np.log1p(yLabels)
 gbm.fit(dataTrain,yLabelsLog)
 preds = gbm.predict(X= dataTrain)
 print("RMSLE Value For Gradient Boost: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))<save_to_csv>
def newage(cols):
title=cols[0]
Sex=cols[1]
Age=cols[2]
if pd.isnull(Age):
if title=='Master' and Sex=="male":
return 5.48
elif title=='Miss' and Sex=='female':
return 21.80
elif title=='Mr' and Sex=='male':
return 32.25
elif title=='Mrs' and Sex=='female':
return 36.87
elif title=='officer' and Sex=='female':
return 49
elif title=='Officer' and Sex=='male':
return 46.14
elif title=='Royalty' and Sex=='female':
return 40
else:
return 42.33
else:
return Age
submission = pd.DataFrame({
"datetime": datetimecol,
"count": [max(0, x)for x in np.exp(predsTest)]
})
 submission.to_csv('bike_predictions_gbm_separate_without_fe.csv', index=False )<set_options>
all_data.Age = all_data[['honorific','Sex','Age']].apply(newage,axis=1 )
%matplotlib inline
sns.set()<load_from_csv>
all_data.family_name.value_counts().head(10 )
df = pd.read_csv(".. /input/train.csv" )<drop_column>
def family_mean(series):
others_survival = pd.Series([series.drop(i ).mean() for i in series.index])
 return others_survival
drop_lst = ['casual', 'registered']
df = df.drop(drop_lst, axis=1)
 df.head()<feature_engineering>
all_data.Survived.fillna(mean_survival,inplace=True)
 all_data['family_survival'] = all_data.query('family_size>1' ).groupby('family_name' ).Survived.transform(family_mean)
 all_data['family_survival'].fillna(mean_survival,inplace=True )
df['count_log'] = count_log
df['count_boxcox'] = count_boxcox<data_type_conversions>
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import KFold
from lightgbm import LGBMClassifier
import lightgbm as lgb
import optuna
df['datetime'] = pd.to_datetime(df['datetime'] )<feature_engineering>
all_data_one_hot = pd.get_dummies(all_data[['Survived','Pclass','Sex','Age','Fare','Embarked','family_size','Cabin_class',
'Ticket_first_letter','family_survival']],
columns=['Ticket_first_letter','Cabin_class','Sex','Pclass','Embarked'] )
df['dow'] = df['datetime'].dt.dayofweek
df.head()<feature_engineering>
def objective(trial):
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate':trial.suggest_loguniform('learning_rate',0.01,0.05),
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'min_child_samples': trial.suggest_int('min_child_samples', 10, 60),
'max_depth':trial.suggest_int('max_depth',3,8)
 }
kf = KFold(n_splits=5,shuffle=True,random_state=1)
 accuracy_scores = []
for train_index,valid_index in kf.split(train):
X_train, X_valid = train.drop(columns='Survived' ).iloc[train_index],train.drop(columns='Survived' ).iloc[valid_index]
y_train, y_valid = train.Survived.iloc[train_index],train.Survived.iloc[valid_index]
lgb_train = lgb.Dataset(X_train,y_train)
 gbm = lgb.train(params,lgb_train)
 y_pred = np.round(gbm.predict(X_valid)).astype(int)
 accuracy_scores.append(accuracy_score(y_valid,y_pred))
return np.mean(accuracy_scores)
 study = optuna.create_study(direction='maximize',sampler=optuna.samplers.RandomSampler(seed=1))
optuna.logging.disable_default_handler()
study.optimize(objective, n_trials=100 )
df['month'] = df['datetime'].dt.month
df.head()<feature_engineering>
print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
df['week'] = df['datetime'].dt.week
df.head()<feature_engineering>
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'verbosity': -1,
'boosting_type': 'gbdt'
}
params.update(trial.params )
df['hour'] = df['datetime'].dt.hour
df.head()<feature_engineering>
kf = KFold(n_splits=5,shuffle=True,random_state=6)
 accuracy_scores = []
for train_index,valid_index in kf.split(train):
X_train, X_valid = train.drop(columns='Survived' ).iloc[train_index],train.drop(columns='Survived' ).iloc[valid_index]
y_train, y_valid = train.Survived.iloc[train_index],train.Survived.iloc[valid_index]
lgb_train = lgb.Dataset(X_train,y_train)
 gbm = lgb.train(params,lgb_train,num_boost_round=100)
 y_pred = np.round(gbm.predict(X_valid)).astype(int)
 accuracy_scores.append(accuracy_score(y_valid,y_pred))
df['year'] = df['datetime'].dt.year
df.head()<feature_engineering>
all_train_data = lgb.Dataset(train.drop(columns=['Survived']),label=train.Survived )
df['day'] = df['datetime'].dt.day
df.head()<drop_column>
gbm_final = lgb.train(train_set=all_train_data,
params = params,
num_boost_round=100
)
df = df.drop(labels='datetime', axis=1)
 df.head()<count_values>
y_pred = np.round(gbm_final.predict(test.drop(columns=['Survived'])) ).astype(int )
spring_2011 = int(df['season'][df['season'] == 1][df['year'] == 2011].value_counts())
 summer_2011 = int(df['season'][df['season'] == 2][df['year'] == 2011].value_counts())
 fall_2011 = int(df['season'][df['season'] == 3][df['year'] == 2011].value_counts())
 winter_2011 = int(df['season'][df['season'] == 4][df['year'] == 2011].value_counts())
 spring_2012 = int(df['season'][df['season'] == 1][df['year'] == 2012].value_counts())
 summer_2012 = int(df['season'][df['season'] == 2][df['year'] == 2012].value_counts())
 fall_2012 = int(df['season'][df['season'] == 3][df['year'] == 2012].value_counts())
 winter_2012 =int(df['season'][df['season'] == 4][df['year'] == 2012].value_counts())
 print("Spring 2011: {}".format(spring_2011))
print("Summer 2011: {}".format(summer_2011))
print("Fall 2011: {}".format(fall_2011))
print("Winter 2011: {}".format(winter_2011))
print("-----------------------------------------")
 print("Spring 2012: {}".format(spring_2012))
print("Summer 2012: {}".format(summer_2012))
print("Fall 2012: {}".format(fall_2012))
print("Winter 2012: {}".format(winter_2012))<count_values>
submission_df = pd.read_csv('.. /input/titanic/gender_submission.csv' )
no_holiday_2011 = int(df['holiday'][df['holiday'] == 0][df['year'] == 2011].value_counts())
 holiday_2011 = int(df['holiday'][df['holiday'] == 1][df['year'] == 2011].value_counts())
 no_holiday_2012 = int(df['holiday'][df['holiday'] == 0][df['year'] == 2012].value_counts())
 holiday_2012 = int(df['holiday'][df['holiday'] == 1][df['year'] == 2012].value_counts())
 print("No Holidays 2011: {}".format(no_holiday_2011))
print("No Holidays 2012: {}".format(no_holiday_2012))
print("Holidays 2011: {}".format(holiday_2011))
print("Holidays 2012: {}".format(holiday_2012))
print('----------------')
 total_2011 = no_holiday_2011 + holiday_2011
total_2012 = no_holiday_2012 + holiday_2012
print('No Holidays 2011: {:.0f}%'.format(no_holiday_2011 / total_2011 * 100))
print('No Holidays 2012: {:.0f}%'.format(no_holiday_2012 / total_2012 * 100))<count_values>
submission_df['Survived'] = y_pred
no_workingday_2011 = int(df['workingday'][df['workingday'] == 0][df['year'] == 2011].value_counts())
 workingday_2011 = int(df['workingday'][df['workingday'] == 1][df['year'] == 2011].value_counts())
 no_workingday_2012 = int(df['workingday'][df['workingday'] == 0][df['year'] == 2012].value_counts())
 workingday_2012 = int(df['workingday'][df['workingday'] == 1][df['year'] == 2012].value_counts())
 print("No working day 2011: {}".format(no_workingday_2011))
print("working day 2011: {}".format(workingday_2011))
print("No working day 2012: {}".format(no_workingday_2012))
print("working day 2012: {}".format(workingday_2012))
print('----------------')
 total_2011 = no_workingday_2011 + workingday_2011
total_2012 = no_workingday_2012 + workingday_2012
print('No working day 2011: {:.0f}%'.format(no_workingday_2011 / total_2011 * 100))
print('No working day 2012: {:.0f}%'.format(no_workingday_2012 / total_2012 * 100))<count_values>
JST = timezone(timedelta(hours=+9), 'JST')
 ts = datetime.now(JST ).strftime('%y%m%d%H%M')
 submission_df.to_csv(( ts+'lgb.csv'),index=False )
weather_2011_1 = df['weather'][df['weather'] == 1][df['year'] == 2011].value_counts()
weather_2011_2 = df['weather'][df['weather'] == 2][df['year'] == 2011].value_counts()
weather_2011_3 = df['weather'][df['weather'] == 3][df['year'] == 2011].value_counts()
weather_2012_1 = df['weather'][df['weather'] == 1][df['year'] == 2012].value_counts()
weather_2012_2 = df['weather'][df['weather'] == 2][df['year'] == 2012].value_counts()
weather_2012_3 = df['weather'][df['weather'] == 3][df['year'] == 2012].value_counts()
weather_2012_4 = df['weather'][df['weather'] == 4][df['year'] == 2012].value_counts()
print('weather_1 in 2011: {}'.format(int(weather_2011_1)))
 print('weather_2 in 2011: {}'.format(int(weather_2011_2)))
 print('weather_3 in 2011: {}'.format(int(weather_2011_3)))
 print('--------------')
 print('weather_1 in 2012: {}'.format(int(weather_2012_1)))
 print('weather_2 in 2012: {}'.format(int(weather_2012_2)))
 print('weather_3 in 2012: {}'.format(int(weather_2012_3)))
 print('weather_4 in 2012: {}'.format(int(weather_2012_4)))
 print('---------------')
 total_2011 = int(weather_2011_1)+ int(weather_2011_2)+ int(weather_2011_3)
 total_2012 = int(weather_2012_1)+ int(weather_2012_2)+ int(weather_2012_3)+ int(weather_2012_4)
 print('weather_1 in 2011: {:.0f}%'.format(int(weather_2011_1)/ int(total_2011)* 100))
print('weather_2 in 2011: {:.0f}%'.format(int(weather_2011_2)/ int(total_2011)* 100))
print('weather_3 in 2011: {:.0f}%'.format(int(weather_2011_3)/ int(total_2011)* 100))
print('--------------')
 print('weather_1 in 2012: {:.0f}%'.format(int(weather_2012_1)/ int(total_2012)* 100))
print('weather_2 in 2012: {:.0f}%'.format(int(weather_2012_2)/ int(total_2012)* 100))
print('weather_3 in 2012: {:.0f}%'.format(int(weather_2012_3)/ int(total_2012)* 100))
print('weather_4 in 2012: {:.0f}%'.format(int(weather_2012_4)/ int(total_2012)* 100))<count_values>
kf = KFold(n_splits=5,shuffle=True,random_state=6)
 mean_accuracy = {}
thresh_holds = np.arange(0.2,0.6,0.01)
 for thresh_hold in thresh_holds:
accuracy_scores = []
for train_index,valid_index in kf.split(train):
X_train, X_valid = train.drop(columns='Survived' ).iloc[train_index],train.drop(columns='Survived' ).iloc[valid_index]
y_train, y_valid = train.Survived.iloc[train_index],train.Survived.iloc[valid_index]
lgb_train = lgb.Dataset(X_train,y_train)
 gbm = lgb.train(params,lgb_train,num_boost_round=100)
 y_pred = np.where(gbm.predict(X_valid)<thresh_hold,0,1)
 accuracy_scores.append(accuracy_score(y_valid,y_pred))
mean_accuracy[thresh_hold] = np.mean(accuracy_scores )