Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<drop_column>
%matplotlib inline
sns.set_style('whitegrid')
 warnings.filterwarnings('ignore')
train_data.drop("Embarked", axis=1, inplace=True)
 test_data.drop("Embarked", axis=1, inplace=True )<prepare_x_and_y>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
ss = StandardScaler()
train_y = train_data["Survived"]
train_data.drop("Survived", axis=1, inplace=True)
 scoring_method = "f1"
train_scaled = ss.fit_transform(train_data)
 test_scaled = ss.transform(test_data )<count_missing_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
print(train_data.isna().sum())
 print(test_data.isna().sum() )<define_search_space>
train['Cabin'].value_counts().head()
model = LogisticRegression(random_state=10, max_iter = 1000)
 logit_params = {
"C": [1, 3, 10, 20, 30, 40],
"solver": ["lbfgs", "liblinear"]
}
logit_gs = GridSearchCV(model, logit_params, scoring="f1", cv = 5, n_jobs=4 )<train_model>
def fill_age_train(cols):
Age = cols[0]
PClass = cols[1]
if pd.isnull(Age):
if PClass == 1:
return 37
elif PClass == 2:
return 29
else:
return 24
else:
return Age
def fill_age_test(cols):
Age = cols[0]
PClass = cols[1]
if pd.isnull(Age):
if PClass == 1:
return 39
elif PClass == 2:
return 28
else:
return 25
else:
return Age
logit_gs.fit(train_data, train_y )<find_best_score>
train['Age'] = train[['Age','Pclass']].apply(fill_age_train,axis=1)
 test['Age'] = test[['Age','Pclass']].apply(fill_age_test,axis=1 )
print(logit_gs.best_params_)
 print(logit_gs.best_score_ )<choose_model_class>
test['Fare'].fillna(stat.mode(test['Fare']),inplace=True)
 train['Embarked'].fillna('S',inplace=True)
 train['Cabin'].fillna('No Cabin',inplace=True)
 test['Cabin'].fillna('No Cabin',inplace=True )
rf_model = RandomForestClassifier()
rf_params ={
'bootstrap': [True, False],
'max_depth': [10, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [5, 10, 15, 20, 25, 30]}
rf_gs = GridSearchCV(rf_model, rf_params, scoring=scoring_method, cv=8, n_jobs=4 )<train_model>
train.drop('Ticket',axis=1,inplace=True)
 test.drop('Ticket',axis=1,inplace=True )
rf_gs.fit(train_data, train_y )<find_best_score>
train['IsTrain'] = 1
test['IsTrain'] = 0
df = pd.concat([train,test] )
print(rf_gs.best_params_)
 print(rf_gs.best_score_ )<choose_model_class>
scaler = MinMaxScaler()
svc_model = SVC()
test_parameters = {
"C": [1, 3, 10, 30, 100],
"kernel": ["linear", "poly", "rbf" , "sigmoid"],
}
svc_gs = GridSearchCV(svc_model, test_parameters, scoring="f1", cv=5, n_jobs=4 )<train_model>
df['Title'] = df['Name'].str.split(', ' ).str[1].str.split('.' ).str[0]
df['Title'].value_counts()
svc_gs.fit(train_scaled, train_y )<find_best_params>
df['Title'].replace('Mme','Mrs',inplace=True)
 df['Title'].replace(['Ms','Mlle'],'Miss',inplace=True)
 df['Title'].replace(['Dr','Rev','Col','Major','Dona','Don','Sir','Lady','Jonkheer','Capt','the Countess'],'Others',inplace=True)
 df['Title'].value_counts()
print(svc_gs.best_params_)
 print(svc_gs.best_score_ )<choose_model_class>
df.drop('Name',axis=1,inplace=True)
 df.head()
lgb_model = LGBMClassifier()
test_parameters = {
"n_estimators": [int(x)for x in np.linspace(5, 30, 6)],
"reg_alpha": [0, 0.75, 1, 1.25],
"learning_rate": [0.5, 0.4, 0.35, 0.3, 0.25, 0.2],
"subsample": [0.5, 0.75, 1]
}
lgb_gs = GridSearchCV(lgb_model, test_parameters, scoring=scoring_method, cv=8, n_jobs=4 )<train_model>
df['AgeGroup'] = df['Age']
df.loc[df['AgeGroup']<=19, 'AgeGroup'] = 0
df.loc[(df['AgeGroup']>19)&(df['AgeGroup']<=30), 'AgeGroup'] = 1
df.loc[(df['AgeGroup']>30)&(df['AgeGroup']<=45), 'AgeGroup'] = 2
df.loc[(df['AgeGroup']>45)&(df['AgeGroup']<=63), 'AgeGroup'] = 3
df.loc[df['AgeGroup']>63, 'AgeGroup'] = 4
lgb_gs.fit(train_data, train_y )<find_best_score>
df.drop('Age',axis=1,inplace=True)
 df.head()
print(lgb_gs.best_params_)
 print(lgb_gs.best_score_ )<choose_model_class>
df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['IsAlone'] = 0
df.loc[df['FamilySize']==1, 'IsAlone'] = 1
ensemble_model = VotingClassifier(estimators=[ 
("logit", logit_gs.best_estimator_), 
("rf", rf_gs.best_estimator_), 
("svc", svc_gs.best_estimator_), 
("lgb", lgb_gs.best_estimator_),
], voting = "hard" )<train_model>
df.drop(['SibSp','Parch','FamilySize'],axis=1,inplace=True)
 df.head()
ensemble_model.fit(train_data, train_y )<compute_test_metric>
df['Deck'] = df['Cabin']
df.loc[df['Deck']!='No Cabin','Deck'] = df[df['Cabin']!='No Cabin']['Cabin'].str.split().apply(lambda x: np.sort(x)).str[0].str[0]
df.loc[df['Deck']=='No Cabin','Deck'] = 'N/A'
ensemble_model.score(train_data, train_y )<predict_on_test>
df.loc[df['Deck']=='N/A', 'Deck'] = 0
df.loc[df['Deck']=='G', 'Deck'] = 1
df.loc[df['Deck']=='F', 'Deck'] = 2
df.loc[df['Deck']=='E', 'Deck'] = 3
df.loc[df['Deck']=='D', 'Deck'] = 4
df.loc[df['Deck']=='C', 'Deck'] = 5
df.loc[df['Deck']=='B', 'Deck'] = 6
df.loc[df['Deck']=='A', 'Deck'] = 7
df.loc[df['Deck']=='T', 'Deck'] = 0
preds = ensemble_model.predict(test_data )<save_to_csv>
df.drop('Cabin',axis=1,inplace=True)
 df.head()
output = pd.DataFrame({'PassengerId': test_data.index,
'Survived': preds})
 output.to_csv('submission.csv', index=False )<import_modules>
df[['Fare','Pclass','Deck']] = scaler.fit_transform(df[['Fare','Pclass','Deck']] )
warnings.filterwarnings("ignore")
<load_from_csv>
def process_dummies(df,cols):
for col in cols:
dummies = pd.get_dummies(df[col],prefix=col,drop_first=True)
 df = pd.concat([df.drop(col,axis=1),dummies],axis=1)
 return df
train=pd.read_csv('.. /input/titanic/train.csv')
 test=pd.read_csv('.. /input/titanic/test.csv')
 y_test=pd.read_csv('.. /input/titanic/gender_submission.csv' )<drop_column>
df = process_dummies(df,['Embarked','Sex','Title','AgeGroup'] )
y_test.head()
PassengerId=y_test['PassengerId']
y_test=y_test.drop(['PassengerId'],axis=1)
<data_type_conversions>
dataset = df[df['IsTrain']==1]
dataset.drop(['IsTrain','PassengerId'],axis=1,inplace=True)
 holdout = df[df['IsTrain']==0]
test_id = holdout['PassengerId']
holdout.drop(['IsTrain','PassengerId','Survived'],axis=1,inplace=True )
train_m=(max(train['Age'])+min(train['Age'])) /2
values={'Cabin':'nocabin','Age':train_m,'Embarked':'notknown'}
train=train.fillna(value=values)
 test_m=(max(test['Age'])+min(test['Age'])) /2
print(test_m)
 values={'Cabin':'nocabin','Age':test_m,'Embarked':'notknown',"Fare":max(test['Fare'])}
test=test.fillna(value=values)
<prepare_x_and_y>
class_one_total = int(np.sum(dataset['Survived']))
class_zero_counter = 0
indices_to_remove = []
for i in range(dataset.shape[0]):
if(dataset['Survived'].iloc[i] == 0):
class_zero_counter += 1
if(class_zero_counter > class_one_total):
indices_to_remove.append(i)
y= train["Survived"]
train=train.drop(["Survived"],axis=1 )<split>
df.to_csv('titanic_dataset_preprocessed.csv',index=False )
X_train, X_cv, y_train, y_cv = train_test_split(train, y, stratify=y, test_size=0.2 )<categorify>
X = dataset.drop(['Survived'],axis=1)
 y = dataset['Survived'].astype('int')
vectorizer = CountVectorizer()
X_tr_emb =vectorizer.fit_transform(X_train['Embarked'])
 X_cv_emb =vectorizer.transform(X_cv['Embarked'])
 X_te_emb =vectorizer.transform(test['Embarked'])
 enc = OneHotEncoder(handle_unknown='ignore')
 X_tr_age =enc.fit_transform(np.array(X_train['Age'] ).reshape(-1,1))
X_cv_age =enc.transform(np.array(X_cv['Age'] ).reshape(-1,1))
X_te_age =enc.transform(np.array(test['Age'] ).reshape(-1,1))
X_tr_fare =enc.fit_transform(np.array(X_train['Fare'] ).reshape(-1,1))
X_cv_fare =enc.transform(np.array(X_cv['Fare'] ).reshape(-1,1))
X_te_fare =enc.transform(np.array(test['Fare'] ).reshape(-1,1))
X_tr_Sbp =enc.fit_transform(np.array(X_train['SibSp'] ).reshape(-1,1))
X_cv_Sbp =enc.transform(np.array(X_cv['SibSp'] ).reshape(-1,1))
X_te_Sbp =enc.transform(np.array(test['SibSp'] ).reshape(-1,1))
X_tr_par =enc.fit_transform(np.array(X_train['Parch'] ).reshape(-1,1))
X_cv_par =enc.transform(np.array(X_cv['Parch'] ).reshape(-1,1))
X_te_par=enc.transform(np.array(test['Parch'] ).reshape(-1,1))
X_tr_pclass =enc.fit_transform(np.array(X_train['Pclass'] ).reshape(-1,1))
X_cv_pclass =enc.transform(np.array(X_cv['Pclass'] ).reshape(-1,1))
X_te_pclass =enc.transform(np.array(test['Pclass'] ).reshape(-1,1))
X_tr_sex =vectorizer.fit_transform(X_train['Sex'])
 X_cv_sex =vectorizer.transform(X_cv['Sex'])
 X_te_sex =vectorizer.transform(test['Sex'])
 X_tr_cabin =vectorizer.fit_transform(X_train['Cabin'])
 X_cv_cabin =vectorizer.transform(X_cv['Cabin'])
 X_te_cabin =vectorizer.transform(test['Cabin'])
 X_tr_tkt =vectorizer.fit_transform(X_train['Ticket'])
 X_cv_tkt =vectorizer.transform(X_cv['Ticket'])
 X_te_tkt =vectorizer.transform(test['Ticket'])


 X_tr = hstack(( X_tr_age,X_tr_fare,X_tr_sex,X_tr_pclass,X_tr_emb)).tocsr()
X_cv = hstack(( X_cv_age,X_cv_fare,X_cv_sex,X_cv_pclass,X_cv_emb)).tocsr()
X_te = hstack(( X_te_age,X_te_fare,X_te_sex,X_te_pclass,X_te_emb)).tocsr()
print(X_tr.shape)
 print(X_te.shape)
 print(X_cv.shape )<train_on_grid>
model = RandomForestClassifier()
kf = KFold(n_splits=10,shuffle=True,random_state=101)
 score = 0
train_indices, validation_indices = [],[]
for curr_train_indices, curr_validation_indices in kf.split(X):
result = model.fit(X.iloc[curr_train_indices], y.iloc[curr_train_indices])
 curr_score = result.score(X.iloc[curr_validation_indices],y.iloc[curr_validation_indices])
 print(curr_score)
 if(curr_score > score):
score = curr_score
train_indices = curr_train_indices
validation_indices = curr_validation_indices
print('Best Score: ',score )
max_depth=[1,5,10,15]
alpha=[1,3,7,15,24,31,60]
cv_log_error_array=[]
min_i=0
val=0
v=99999
for i in alpha:
clf = KNeighborsClassifier(n_neighbors=i,weights='distance')
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_cv)
 val=log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15)
 if val<v:
min_i=i
v=val
cv_log_error_array.append(val)
 print('For values of alpha = ', i,"The log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
clf = KNeighborsClassifier(n_neighbors=i,weights='distance')
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_tr)
 print('For values of best alpha = ',min_i, "The train log loss is:",log_loss(y_train, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_cv)
 print('For values of best alpha = ',min_i, "The cross validation log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_te)
 print('For values of best alpha = ',min_i, "The test log loss is:",log_loss(y_test, predict_y, labels=clf.classes_, eps=1e-15))
knn=min_i<save_to_csv>
param_grid = [
{'n_estimators':[1,10,20,50,100,1000,3000], 'min_samples_leaf':[1,2,3,4,5], 'max_features':[3,5,7,9,10,'auto']},
]
grid = GridSearchCV(model,param_grid,n_jobs=4)
 grid.fit(X, y )
pred=sig_clf.predict(X_te)
 df=pd.DataFrame(zip(PassengerId,pred),columns=['PassengerId',"Survived"])
 df
df.to_csv('/kaggle/working/knn_output.csv',index=False)
<load_from_csv>
grid.best_params_, grid.best_score_
d=pd.read_csv('/kaggle/working/knn_output.csv')
 d<set_options>
predictions = grid.predict(holdout )
<load_from_csv><EOS> 
submission = pd.DataFrame({
'PassengerId': test_id,
'Survived': predictions
})
 submission.to_csv('submission.csv',index=False )