Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_unique_values>
%matplotlib inline
warnings.filterwarnings('ignore' )
addr=['addr1','addr2','dist1','dist2','P_emaildomain','R_emaildomain']
for c in addr:
print(f'Number of unique variables in {c}: ',train[c].nunique() )<count_unique_values>
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv")
 target = train["Survived"]
data = pd.concat([train.drop("Survived", axis=1),test], axis=0 ).reset_index(drop=True )
C_columns=[col for col in train.columns if 'C'==col[0]]
for c in C_columns:
print(f'Number of unique entries in {c}:',train[c].nunique() )<count_unique_values>
data.isnull().sum()
D_cols=[col for col in train.columns if col[0]=='D']
for d in D_cols:
print(f'Number of unique entries in {d}:',train[d].nunique() )<count_values>
data.isnull().sum()
print(train['D9'].value_counts())
 print('*'*100)
 print(test['D9'].value_counts() )<count_unique_values>
data["Family_Size"] = data["SibSp"] + data["Parch"]
missing_age_value = data[data["Age"].isnull() ]
for index, row in missing_age_value.iterrows() :
median = data["Age"][(data["Pclass"] == row["Pclass"])&(data["Embarked"] == row["Embarked"])&(data["Family_Size"] == row["Family_Size"])].median()
if not np.isnan(median):
data["Age"][index] = median
else:
data["Age"][index] = np.median(data["Age"].dropna() )
M_cols=[col for col in train.columns if col[0]=='M']
for m in M_cols:
print(f'Number of unique entries in {m}:',train[m].nunique() )<define_variables>
for index, rows in data.iterrows() :
if pd.isnull(rows["Cabin"]):
data["Cabin"][index] = 'X'
else:
data["Cabin"][index] = str(rows["Cabin"])[0]
V_cols=[col for col in train.columns if col[0]=='V']
V_type=[train[col].dtype for col in V_cols]
print(set(V_type))
print(len(V_cols))<count_unique_values>
data["Embarked"].fillna(data["Embarked"].mode() [0], inplace=True )
unique=[(col,train[col].nunique())for col in V_cols]
sorted_unique=sorted(unique,key=lambda x: x[1])
 sorted_unique[:50]<count_unique_values>
data.drop(["SibSp", "Parch"], inplace=True, axis=1)
 data = pd.get_dummies(data=data, columns=["Pclass", "Name", "Sex", "Age", "Cabin", "Embarked", "Family_Size", "Ticket", "Fare"], drop_first=True )
i_cols=[col for col in train.columns if col[0]=='i']
unique_val=[(col,train[col].nunique())for col in i_cols]
unique_val<define_variables>
train = data[:891]
test = data[891:]
drop_cols=['dist2','C8','C9','C10','D6','D12','V27','V28','V45','V70','V71','V77','V78','V86','V87','V89','V91','V92','V95','V96','V97',
'V101','V102','V103','V107','V126','V127','V128','V130','V131','V132','V133','V134','V135','V136','V137','V143','V145','V150',
'V159','V160','V164','V165','V166','V167','V168','V171','V176','V177','V178','V179','V181','V182','V183','V190','V199','V202',
'V203','V204','V207','V211','V212','V213','V214','V215','V216','V221','V226','V227','V228','V230','V234','V240','V241','V245',
'V246','V255','V256','V257','V258','V259','V261','V263','V264','V265','V270','V271','V272','V273','V274','V275','V276','V277',
'V278','V279','V280','V291','V292','V293','V294','V295','V297','V306','V307','V308','V310','V312','V316','V317','V318','V319',
'V320','V321','V322','V323','V324','V331','V332','V333','V338','V339','TransactionDT','TransactionID']
train=train.drop(drop_cols,1)
 test=test.drop(drop_cols,1)
 train['TransactionAmt']=np.log(train['TransactionAmt'])
 test['TransactionAmt']=np.log(test['TransactionAmt'] )<define_variables>
cv_split = model_selection.ShuffleSplit(n_splits=10, test_size=.3, train_size=.7, random_state=42)
 classifiers = [
SVC(random_state=42),
DecisionTreeClassifier(random_state=42),
AdaBoostClassifier(DecisionTreeClassifier(random_state=42),random_state=42,learning_rate=0.1),
RandomForestClassifier(random_state=42),
ExtraTreesClassifier(random_state=42),
GradientBoostingClassifier(random_state=42),
MLPClassifier(random_state=42),
KNeighborsClassifier() ,
LogisticRegression(random_state=42),
LinearDiscriminantAnalysis()
]
cv_train_mean = []
cv_test_mean = []
cv_score_time = []
cv_fit_time = []
cv_name = []
predictions = []
for classifier in classifiers :
cv_results = model_selection.cross_validate(classifier, train.drop(['PassengerId'], axis=1), target, cv=cv_split, return_train_score=True)
 cv_train_mean.append(cv_results['train_score'].mean())
 cv_test_mean.append(cv_results['test_score'].mean())
 cv_score_time.append(cv_results['score_time'].mean())
 cv_fit_time.append(cv_results['fit_time'].mean())
 cv_name.append(str(classifier.__class__.__name__))
classifier.fit(train.drop(['PassengerId'], axis=1), target)
 predictions.append(classifier.predict(test.drop(['PassengerId'], axis=1)))
 performance_df = pd.DataFrame({"Algorithm":cv_name, "Train Score":cv_train_mean, "Test Score":cv_test_mean, 'Score Time':cv_score_time, 'Fit Time':cv_fit_time})
 performance_df
<statistical_test>
tuned_clf = {
'DecisionTreeClassifier':DecisionTreeClassifier(random_state=42),
'AdaBoostClassifier':AdaBoostClassifier(DecisionTreeClassifier(random_state=42),random_state=42,learning_rate=0.1),
'RandomForestClassifier':RandomForestClassifier(random_state=42),
'ExtraTreesClassifier':ExtraTreesClassifier(random_state=42),
'GradientBoostingClassifier':GradientBoostingClassifier(random_state=42),
'MLPClassifier':MLPClassifier(random_state=42),
'LogisticRegression':LogisticRegression(random_state=42),
'LinearDiscriminantAnalysis':LinearDiscriminantAnalysis()
}
train=reduce_mem_usage(train)
 test=reduce_mem_usage(test )<count_unique_values>
grid = {'criterion': ['gini', 'entropy'], 'splitter': ['best', 'random'], 'max_depth': [2,4,6,8,10,None],
'min_samples_split': [2,5,10,.03,.05], 'min_samples_leaf': [1,5,10,.03,.05], 'max_features': [None, 'auto']}
tune_model = model_selection.GridSearchCV(tuned_clf['DecisionTreeClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['DecisionTreeClassifier'].set_params(**tune_model.best_params_ )
<categorify>
grid = {'n_estimators': [10, 50, 100, 300], 'learning_rate': [.01,.03,.05,.1,.25], 'algorithm': ['SAMME', 'SAMME.R'] }
tune_model = model_selection.GridSearchCV(tuned_clf['AdaBoostClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['AdaBoostClassifier'].set_params(**tune_model.best_params_ )
<feature_engineering>
grid = {'n_estimators': [10, 50, 100, 300], 'criterion': ['gini', 'entropy'], 'max_depth': [2, 4, 6, 8, 10, None],
'oob_score': [True] }
tune_model = model_selection.GridSearchCV(tuned_clf['RandomForestClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['RandomForestClassifier'].set_params(**tune_model.best_params_ )
num_cols=[col for col in train.columns if(( 'float' in str(train[col].dtype)) or('int' in str(train[col].dtype)))]
num_cols.remove('isFraud')
 train['mean']=train[num_cols].mean(axis=1)
 test['mean']=test[num_cols].mean(axis=1)
 train['std']=train[num_cols].std(axis=1)
 test['std']=test[num_cols].std(axis=1)
 train['max']=train[num_cols].max(axis=1)
 test['max']=test[num_cols].max(axis=1)
 train['min']=train[num_cols].min(axis=1)
 test['min']=test[num_cols].min(axis=1)
 train['median']=train[num_cols].median(axis=1)
 test['median']=test[num_cols].median(axis=1)
 train['skew']=train[num_cols].skew(axis=1)
 test['skew']=test[num_cols].skew(axis=1)
 train['kurt']=train[num_cols].kurt(axis=1)
 test['kurt']=test[num_cols].kurt(axis=1 )<categorify>
grid = {'n_estimators': [10, 50, 100, 300], 'criterion': ['gini', 'entropy'], 'max_depth': [2, 4, 6, 8, 10, None]}
tune_model = model_selection.GridSearchCV(tuned_clf['ExtraTreesClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['ExtraTreesClassifier'].set_params(**tune_model.best_params_ )
<data_type_conversions>
grid = {
'n_estimators': [300],
'max_depth': [4] }
tune_model = model_selection.GridSearchCV(tuned_clf['GradientBoostingClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['GradientBoostingClassifier'].set_params(**tune_model.best_params_ )
<categorify>
grid = {'learning_rate': ["constant", "invscaling", "adaptive"], 'alpha': 10.0 ** -np.arange(1, 7), 'activation': ["logistic", "relu", "tanh"]}
tune_model = model_selection.GridSearchCV(tuned_clf['MLPClassifier'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['MLPClassifier'].set_params(**tune_model.best_params_ )
object_cols=[col for col in train.columns if(( 'category' in str(train[col].dtype)) or('object' in str(train[col].dtype)))]
le=LabelEncoder()
for col in object_cols:
le.fit(list(train[col].values)+list(test[col].values))
train[col]=le.transform(list(train[col].values))
test[col]=le.transform(list(test[col].values))<prepare_x_and_y>
grid = {'fit_intercept': [True, False], 'penalty': ['l2'], 'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']}
tune_model = model_selection.GridSearchCV(tuned_clf['LogisticRegression'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['LogisticRegression'].set_params(**tune_model.best_params_ )
target=train['isFraud']
train=train.drop('isFraud',1 )<train_model>
grid = {"solver" : ["svd"], "tol" : [0.0001,0.0002,0.0003]}
tune_model = model_selection.GridSearchCV(tuned_clf['LinearDiscriminantAnalysis'], param_grid=grid, scoring = 'roc_auc', cv = cv_split, return_train_score=True)
 tune_model.fit(train.drop(['PassengerId'], axis=1), target)
 print("Best Parameters:")
 print(tune_model.best_params_)
 tuned_clf['LinearDiscriminantAnalysis'].set_params(**tune_model.best_params_ )
<init_hyperparams>
cv_train_mean = []
cv_test_mean = []
cv_score_time = []
cv_fit_time = []
cv_name = []
predictions = []
for _, classifier in tuned_clf.items() :
cv_results = model_selection.cross_validate(classifier, train.drop(['PassengerId'], axis=1), target, cv=cv_split, return_train_score=True)
 cv_train_mean.append(cv_results['train_score'].mean())
 cv_test_mean.append(cv_results['test_score'].mean())
 cv_score_time.append(cv_results['score_time'].mean())
 cv_fit_time.append(cv_results['fit_time'].mean())
 cv_name.append(str(classifier.__class__.__name__))
classifier.fit(train.drop(['PassengerId'], axis=1), target)
 predictions.append(classifier.predict(test.drop(['PassengerId'], axis=1)))
 performance_df = pd.DataFrame({"Algorithm":cv_name, "Train Score":cv_train_mean, "Test Score":cv_test_mean, 'Score Time':cv_score_time, 'Fit Time':cv_fit_time})
 performance_df
nfolds=10
xgb_params=dict(n_estimators=1000,
verbosity=0,
tree_method='gpu_hist',
random_state=seed,
colsample_bytree=0.6,
subsample=0.6,
learning_rate=0.05,
max_depth=9)
 lgb_params=dict(objective='binary',
num_leaves=62,
seed=seed,
max_depth=9,
pos_bagging_fraction=0.5,
neg_bagging_fraction=1.0,
bagging_freq=5,
feature_fraction=0.9,
metric='auc',
learning_rate=0.05,
verbosity=-1,
device='gpu')
 skfold=StratifiedKFold(nfolds,random_state=seed)
 def build_model(params,model='xgb',plot_feature_importance=True):
oof=np.zeros(train.shape[0])
 pred=np.zeros(test.shape[0])
 scores=[]
feature_importance=pd.DataFrame()
for i,(train_index,val_index)in enumerate(skfold.split(train,target)) :
print('Fold :',i+1)
 if model=='xgb':
train_X,val_X=train.iloc[train_index,:],train.iloc[val_index,:]
train_y,val_y=target[train_index],target[val_index]
clf=XGBClassifier(**params)
 clf.fit(train_X,train_y,eval_metric='auc',eval_set=[(val_X,val_y)],early_stopping_rounds=10,verbose=20)
 val_pred=clf.predict_proba(val_X)[:,1]
if model=='lgb':
train_d=lgb.Dataset(train.iloc[train_index,:].values,label=target[train_index].values)
 val_d=lgb.Dataset(train.iloc[val_index,:].values,label=target[val_index].values)
 clf=lgb.train(params,train_d,num_boost_round=1000,valid_sets=[val_d],verbose_eval=20,early_stopping_rounds=10)
 val_pred=clf.predict(train.iloc[val_index,:].values)
 oof[val_index]=val_pred
val_score=roc_auc_score(target[val_index],val_pred)
 scores.append(val_score)
 print(f'Validation score using {model} for fold {i} :'+ str(val_score))
print('-'*100)
 if model=='xgb':
pred+=clf.predict_proba(test)[:,1]/nfolds
if model=='lgb':
pred+=clf.predict(test.values)/nfolds
if model=='xgb':
del train_X,val_X,train_y,val_y
if model=='lgb':
del train_d,val_d
gc.collect()
fold_importance=pd.DataFrame()
fold_importance['feature']=train.columns
if model=='xgb':
fold_importance['importance']=clf.feature_importances_
if model=='lgb':
fold_importance['importance']=clf.feature_importance()
fold_importance['fold']=i+1
feature_importance=pd.concat([feature_importance,fold_importance],axis=0)
 print('Mean validation score :',np.mean(scores))
if plot_feature_importance:
df=feature_importance[['feature','importance']].groupby('feature' ).mean().sort_values(by='importance',ascending=False ).reset_index()
plt.figure(figsize=(10,10))
sns.barplot(x='importance',y='feature',data=df.iloc[:25,:])
 plt.title('Feature Importances')
 return pred<choose_model_class>
voting_3 = VotingClassifier(estimators=[ 
('DecisionTreeClassifier', tuned_clf['DecisionTreeClassifier']), 
('AdaBoostClassifier', tuned_clf['AdaBoostClassifier']), 
('RandomForestClassifier', tuned_clf['RandomForestClassifier']), 
('ExtraTreesClassifier',tuned_clf['ExtraTreesClassifier']), 
('GradientBoostingClassifier',tuned_clf['GradientBoostingClassifier']), 
('MLPClassifier',tuned_clf['MLPClassifier']), 
('LogisticRegression', tuned_clf['LogisticRegression']), 
('LinearDiscriminantAnalysis', tuned_clf['LinearDiscriminantAnalysis'])], voting='soft', n_jobs=4)
 voting_3 = voting_3.fit(train.drop(['PassengerId'], axis=1), target)
 pred_3 = voting_3.predict(test.drop(['PassengerId'], axis=1))
pred=build_model(xgb_params,model='xgb' )<save_to_csv>
sol3 = pd.DataFrame(data=pred_3, columns=['Survived'], index=test['PassengerId'] )