Kaggle competition: Titanic - Machine Learning from Disaster
train['t_preds'] = train_preds<predict_on_test>
%matplotlib inline
random_state=42
warnings.filterwarnings(action='ignore' )
preds = model.predict(val_ds )<save_to_csv>
train=pd.read_csv('.. /input/titanic/train.csv')
 test=pd.read_csv('.. /input/titanic/test.csv' )
sub = pd.DataFrame({'id':test.id, 'target':preds})
 sub.to_csv('submission.csv',index=False )<load_from_csv>
gender=pd.read_csv('.. /input/titanic/gender_submission.csv')
 gender
x = pd.read_csv('./submission.csv')
 x.head()<import_modules>
X=train.drop('Survived',axis=1)
 y=train['Survived']
import random
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.mixture import GaussianMixture
from sklearn.metrics import confusion_matrix
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from lightgbm import LGBMClassifier
from xgboost import XGBRegressor
import optuna
from optuna import Trial
from optuna.samplers import TPESampler<load_from_csv>
x_train,x_val,y_train,y_val=train_test_split(X,y,test_size=0.2,random_state=random_state )
df_train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 df_test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 df_train.info(verbose=False, memory_usage='deep' )<define_variables>
tab=pd.pivot_table(train,index='Sex',values='Survived',aggfunc=np.sum)
 df=pd.DataFrame({'% of survived':[tab.loc['female','Survived']/np.sum(tab)['Survived']*100,tab.loc['male','Survived']/np.sum(tab)['Survived']*100]})
 df.index=tab.index
tab=pd.concat([tab,df],axis=1)
 tab
train_mode = False
target = 'target'
random_state = 42
x_cols = [c for c in df_train.columns if 'cont' in c]<split>
x_train.drop(columns=['Name','Ticket','Cabin'],inplace=True,axis=1)
 x_val.drop(columns=['Name','Ticket','Cabin'],inplace=True,axis=1 )
def _evaluate(model, x_cols, df, target=target, n_folds=5):
oof = np.zeros(len(df[target]))
preds_test = np.zeros(len(df_test))
kf = KFold(n_splits=n_folds, random_state=random_state, shuffle=True)
 for idx_train, idx_test in kf.split(df):
x_train = df.loc[idx_train, x_cols].values
y_train = df.loc[idx_train, target].values
x_test = df.loc[idx_test, x_cols].values
y_test = df.loc[idx_test, target].values
model.fit(x_train, y_train)
 preds_train = model.predict(x_test)
 preds_test += model.predict(df_test[x_cols].values)/ n_folds
oof[idx_test] = preds_train
return oof, preds_test<find_best_params>
impute=SimpleImputer(strategy='most_frequent')
 x_train_1=pd.DataFrame(impute.fit_transform(x_train[['Embarked']]))
x_val_1=pd.DataFrame(impute.transform(x_val[['Embarked']]))
x_train_1.columns=['Embarked']
x_val_1.columns=['Embarked']
x_train_1.index=x_train.index
x_val_1.index=x_val.index
x_train.drop('Embarked',axis=1,inplace=True)
 x_val.drop('Embarked',axis=1,inplace=True)
 x_train=pd.concat([x_train,x_train_1],axis=1)
 x_val=pd.concat([x_val,x_val_1],axis=1)
 x_train
def objective(trial):
param = {
'boosting_type':'gbdt',
'num_leaves':trial.suggest_int('num_leaves', 3, 150),
'max_depth':trial.suggest_int('max_depth', -1, 20),
'learning_rate':trial.suggest_float('learning_rate', 0.001, 0.6),
'n_estimators':trial.suggest_int('n_estimators', 50, 500),
'min_child_weight':trial.suggest_float('min_child_weight', 0.2, 0.6),
'min_child_samples':trial.suggest_int('min_child_samples', 15, 30),
'subsample':trial.suggest_float('subsample', 0.5, 1.0),
'subsample_freq':trial.suggest_int('subsample_freq', 3, 150),
'random_state':random_state,
'lambda_l1':trial.suggest_float('lambda_l1', 0.0, 5.0)
 }
lgbm = LGBMRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(lgbm, x_cols, df_train.reset_index(drop=True))
return mean_squared_error(df_train[target].reset_index(drop=True), oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=200)
 print(study.best_value)
 print(study.best_params )<choose_model_class>
impute_1=SimpleImputer(strategy='mean')
 x_train_2=pd.DataFrame(impute_1.fit_transform(x_train[['Age']]))
x_val_2=pd.DataFrame(impute_1.transform(x_val[['Age']]))
x_train_2.columns=['Age']
x_val_2.columns=['Age']
x_train_2.index=x_train.index
x_val_2.index=x_val.index
x_train.drop('Age',axis=1,inplace=True)
 x_val.drop('Age',axis=1,inplace=True)
 x_train=pd.concat([x_train,x_train_2],axis=1)
 x_val=pd.concat([x_val,x_val_2],axis=1)
 x_train
def objective(trial):
param = {
'iterations':1000,
'verbose':False,
'random_state':random_state,
'loss_function':'RMSE',
'bootstrap_type':'Bernoulli',
'learning_rate':trial.suggest_float('learning_rate', 0.0001, 0.31),
'max_depth':trial.suggest_int('max_depth', 3, 10),
'colsample_bylevel':trial.suggest_float('colsample_bylevel', 0.3, 0.8),
}
cat = CatBoostRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(cat, x_cols, df_train, target=target)
 return mean_squared_error(df_train[target], oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=100)
 print(study.best_value)
 print(study.best_params )<train_model>
print("{} null-values left in training-set".format(x_train.isnull().any().sum()),end='
')
 print("{} null-values left in validation-set".format(x_val.isnull().any().sum()),end='
' )
def objective(trial):
param = {
'random_state':random_state,
'objective':'reg:squarederror',
'booster':'gbtree',
'learning_rate':trial.suggest_float('learning_rate', 0.001, 0.1),
'alpha':trial.suggest_float('alpha', 0.001, 0.1),
'colsample_bylevel':trial.suggest_float('colsample_bylevel', 0.05, 0.1),
'colsample_bytree':trial.suggest_float('colsample_bytree', 0.1, 0.9),
'gamma':trial.suggest_float('gamma', 0, 0.5),
'max_depth':trial.suggest_int('max_depth', 3, 18),
'min_child_weight': trial.suggest_float('min_child_weight', 1, 20),
'subsample':trial.suggest_float('subsample', 0.3, 0.7),
}
xgb = XGBRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(xgb, x_cols, df_train, target=target)
 return mean_squared_error(df_train[target], oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=100)
 print(study.best_value)
 print(study.best_params )<init_hyperparams>
encode=OneHotEncoder(handle_unknown='ignore',sparse=False)
 categorical_cols=list(x_train.select_dtypes(include=['object'] ).columns)
 categorical_cols
x_train_oh=pd.DataFrame(encode.fit_transform(x_train[categorical_cols]))
x_val_oh=pd.DataFrame(encode.transform(x_val[categorical_cols]))
x_train_oh.index=x_train.index
x_val_oh.index=x_val.index
x_train.drop(columns=categorical_cols,axis=1,inplace=True)
 x_val.drop(columns=categorical_cols,axis=1,inplace=True)
 x_train=pd.concat([x_train,x_train_oh],axis=1)
 x_val=pd.concat([x_val,x_val_oh],axis=1)
 x_train
param_lgbm = {'num_leaves': 148, 'max_depth': 19, 'learning_rate': 0.04168752594808129, 'n_estimators': 468, 'min_child_weight': 0.35392113973764505, 'min_child_samples': 29, 'subsample': 0.9348697769228501, 'subsample_freq': 6, 'lambda_l1': 4.639129744838143}
param_cat = {'learning_rate': 0.07468089271003528, 'max_depth': 8, 'colsample_bylevel': 0.7338241468797853}
param_xgb = param_xgb = {'learning_rate': 0.0652222304334701, 'alpha': 0.0036866921576056855, 'colsample_bylevel': 0.09959606060270643, 'colsample_bytree': 0.863554381598069, 'gamma': 0.3959383978062547, 'max_depth': 15, 'min_child_weight': 19.357558021086128, 'subsample': 0.6991638855748524}
lgbm = LGBMRegressor(seed=random_state, **param_lgbm)
 oof_lgbm, preds_lgbm = _evaluate(lgbm, x_cols, df_train, n_folds=5)
 print('hyper lgbm:', mean_squared_error(df_train[target], oof_lgbm, squared=False))
cat = CatBoostRegressor(iterations=1000, verbose=False, random_state=random_state, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 oof_cat, preds_cat = _evaluate(cat, x_cols, df_train, n_folds=5)
 print('hyper cat:', mean_squared_error(df_train[target], oof_cat, squared=False))
xgb = XGBRegressor(random_state=random_state, **param_xgb)
 oof_xgb, preds_xgb = _evaluate(xgb, x_cols, df_train, n_folds=2)
 print('hyper xgb:', mean_squared_error(df_train[target], oof, squared=False))
print('hyper blend:', mean_squared_error(df_train[target],(( oof_lgbm * 0.4)+(oof_cat * 0.4)+(oof_xgb * 0.2)) , squared=False))<save_to_csv>
col_scale=['Age','Fare']
scale=StandardScaler()
x_train_scale=pd.DataFrame(scale.fit_transform(x_train[col_scale]))
x_val_scale=pd.DataFrame(scale.transform(x_val[col_scale]))
x_train_scale.index=x_train.index
x_val_scale.index=x_val.index
x_train_scale.columns=col_scale
x_val_scale.columns=col_scale
x_train.drop(columns=col_scale,axis=1,inplace=True)
 x_val.drop(columns=col_scale,axis=1,inplace=True)
 x_train=pd.concat([x_train,x_train_scale],axis=1)
 x_val=pd.concat([x_val,x_val_scale],axis=1)
 x_train
df_test['hyper_lgbm'] = preds_lgbm
df_test['hyper_cat'] = preds_cat
df_test['hyper_xgb'] = preds_xgb
df_test['hyper_blend'] =(preds_lgbm * 0.4)+(preds_cat * 0.4)+(preds_xgb * 0.2)
 df_test['target'] = df_test['hyper_blend']
df_test[['id','target']].to_csv('submission_hyper_blend.csv', index=False )<init_hyperparams>
test.drop(columns=['Ticket','Cabin','Name'],axis=1,inplace=True )
n_folds = 3
oof_lgbm_seeds = np.zeros(len(df_train))
oof_car_seeds = np.zeros(len(df_train))
preds_lgbm_seeds = np.zeros(len(df_test))
preds_car_seeds = np.zeros(len(df_test))
for g in range(n_folds):
rand_num = random.randint(1, 5000)
 lgbm = LGBMRegressor(seed=rand_num, **param_lgbm)
 oof, preds_lgbm = _evaluate(lgbm, x_cols, df_train, n_folds=n_folds)
 oof_lgbm_seeds +=(oof / n_folds)
 preds_lgbm_seeds +=(preds_lgbm / n_folds)
 cat = CatBoostRegressor(iterations=1000, verbose=False, random_state=rand_num, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 oof, preds_cat = _evaluate(cat, x_cols, df_train, n_folds=n_folds)
 oof_car_seeds +=(oof / n_folds)
 preds_car_seeds +=(preds_cat / n_folds )<compute_test_metric>
test_1=pd.DataFrame(impute_1.transform(test[['Age']]))
test_1.columns=['Age']
test_1.index=test.index
test.drop('Age',axis=1,inplace=True)
 test=pd.concat([test,test_1],axis=1)
 test
print('hyper lgbm:', mean_squared_error(df_train[target], oof_lgbm_seeds, squared=False))
print('hyper cat:', mean_squared_error(df_train[target], oof_car_seeds, squared=False)) 
<save_to_csv>
impute_3=SimpleImputer(strategy='most_frequent')
 test_2=pd.DataFrame(impute_3.fit_transform(test[['Fare']]))
test_2.columns=['Fare']
test_2.index=test.index
test.drop('Fare',axis=1,inplace=True)
 test=pd.concat([test,test_2],axis=1)
 test
df_test['seed_lgbm'] = preds_lgbm_seeds
df_test['seed_cat'] = preds_car_seeds
df_test['seed_blend'] =(preds_lgbm_seeds * 0.4)+(preds_car_seeds * 0.6)
 df_test['target'] = df_test['seed_blend']
df_test[['id','target']].to_csv('submission_seed_blend.csv', index=False )<compute_train_metric>
test_oh=pd.DataFrame(encode.transform(test[categorical_cols]))
test_oh.index=test.index
test.drop(columns=categorical_cols,axis=1,inplace=True)
 test=pd.concat([test,test_oh],axis=1)
 test
lgbm_gmm_class = LGBMClassifier(random_state=random_state)
 oof, preds = _evaluate(lgbm_gmm_class, x_cols, df_gmm_train, target='target_gmm')
 print(accuracy_score(df_gmm_train['target_gmm'], oof))
print(confusion_matrix(df_gmm_train['target_gmm'], oof))<choose_model_class>
test_scale=pd.DataFrame(scale.transform(test[col_scale]))
test_scale.columns=col_scale
test_scale.index=test.index
test.drop(columns=col_scale,axis=1,inplace=True)
 test=pd.concat([test,test_scale],axis=1)
 test
param_lgbm_gmm_class = {'num_leaves': 144, 'max_depth': 10, 'learning_rate': 0.0934759313797003, 'n_estimators': 107, 'min_child_weight': 0.30363944085393396, 'min_child_samples': 17, 'subsample': 0.9463879414095, 'subsample_freq': 80, 'lambda_l1': 4.67131782429971}
lgbm_gmm_class = LGBMClassifier(random_state=random_state, **param_lgbm_gmm_class)
 lgbm_gmm_reg_0 = LGBMRegressor(seed=random_state)
 lgbm_gmm_reg_1 = LGBMRegressor(seed=random_state)
 lgbm_gmm_reg = LGBMRegressor(seed=random_state, **param_lgbm)
 lgbm_cat_reg = CatBoostRegressor(iterations=1000, verbose=False, random_state=random_state, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 mask_gmm_0 = df_gmm_train['target_gmm'] == 0
mask_gmm_1 = df_gmm_train['target_gmm'] == 1
lgbm_gmm_class.fit(df_gmm_train[x_cols], df_gmm_train['target_gmm'])
 lgbm_gmm_reg_0.fit(df_gmm_train.loc[mask_gmm_0, x_cols].reset_index(drop=True), df_gmm_train.loc[mask_gmm_0, target].reset_index(drop=True))
lgbm_gmm_reg_1.fit(df_gmm_train.loc[mask_gmm_1, x_cols].reset_index(drop=True), df_gmm_train.loc[mask_gmm_1, target].reset_index(drop=True))
lgbm_gmm_reg.fit(df_gmm_train[x_cols], df_gmm_train[target])
 lgbm_cat_reg.fit(df_gmm_train[x_cols], df_gmm_train[target])
 df_gmm_test[['gmm_class_' + str(a)+ '_pred' for a in lgbm_gmm_class.classes_]] = lgbm_gmm_class.predict_proba(df_gmm_test[x_cols])
 df_gmm_test['lgbm'] = lgbm_gmm_reg.predict(df_gmm_test[x_cols])
 df_gmm_test['cat'] = lgbm_cat_reg.predict(df_gmm_test[x_cols])
 df_gmm_test['hyper_blend'] =(df_gmm_test['lgbm'] * 0.5)+(df_gmm_test['cat'] * 0.5)
 df_gmm_test['gmm_reg_0'] = lgbm_gmm_reg_0.predict(df_gmm_test[x_cols])
 df_gmm_test['gmm_reg_1'] = lgbm_gmm_reg_1.predict(df_gmm_test[x_cols] )<compute_test_metric>
log_reg=lr(random_state=random_state)
 sgd=sg(random_state=random_state)
 random_forest=rf(random_state=random_state)
 tree_model=dt(random_state=random_state)
 svm_model=SVC(random_state=random_state)
 xgb=XGBClassifier(random_state=random_state)
 models={'Logistic Regression':log_reg,'SGD':sgd,'Random Forest':random_forest,'Decision Tree':tree_model,'SVM':svm_model,'XGBoost':xgb}
for model_name,model in models.items() :
model.fit(x_train,y_train)
 rslt=model.predict(x_val)
 print('{}:{}'.format(model_name,np.sum(rslt==y_val)/len(y_val)) ,end='
' )
print('blend:', mean_squared_error(df_gmm_test[target], df_gmm_test['hyper_blend'], squared=False))
print('super:', mean_squared_error(df_gmm_test[target], df_gmm_test['super_target'], squared=False))<define_variables>
result=random_forest.predict(test )
%matplotlib inline
dfk = pd.DataFrame({
'Kernel ID': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L'],
'Score': [ 0.69864 , 0.69846 , 0.69836 , 0.69824 , 0.69813, 0.69795, 0.69751, 0.69749, 0.69747, 0.69735, 0.69731, 0.69701],
'File Path': ['.. /input/aa69864/AA69864.csv', '.. /input/bb69846/BB69846.csv', '.. /input/cc69836/CC69836.csv', '.. /input/a69824/A69824.csv', '.. /input/c69813/C69813.csv', '.. /input/ff69795/FF69795.csv', '.. /input/gg69751/GG69751.csv' , '.. /input/g69749/G69749.csv', '.. /input/h69747/H69747.csv', '.. /input/i69735/I69735.csv', '.. /input/j69731/J69731.csv', '.. /input/l69701/L69701.csv']
})
 dfk<categorify>
output=pd.DataFrame({'PassengerId':test.PassengerId,
'Survived':result})
 output.to_csv('submission.csv',index=False )