Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.layers import Input, Dense, BatchNormalization, Add, GaussianNoise, Dropout
from keras.models import Model
from keras.layers import Wrapper
from keras.callbacks import ReduceLROnPlateau
from keras.utils import to_categorical
from keras import regularizers, Sequential
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
'learning_rate': 0.01, 'loss': 'ls'}
clfGB = GradientBoostingRegressor(**params)
 clfGB.fit(X_trainD, y_trainD)
 rGB=clfGB.predict(dfteste_d)
 clfRF = RandomForestRegressor()
clfRF.fit(X_trainD, y_trainD)
 rRF=clfRF.predict(dfteste_d)
 params= {'boosting_type' : 'dart',
'max_depth':-1,
'objective':'regression',
'nthread': 5,
'num_leaves':64,
'learning_rate':0.01,
'max_bin':256,
'subsample_for_bin':200,
'subsample':1,
'subsample_freq':1,
'colsample_bytree':0.8,
'reg_alpha':1.2,
'reg_lambda':1.2,
'min_split_gain':0.5,
'min_child_weight':1,
'min_child_samples':5,
'metric':'l2'
}
gridParams = {'learning_rate': [0.001,0.01,0.03,0.05,0.07,0.09]
,'num_leaves':[10,20,30,50,70]
,'boosting_type': ['dart']
,'objective': ['regression']
,'random_state' : [13]
,'min_split_gain': [0.01,0.03,0.05]
,'drop_rate' : [0.015,0.02,0.04,0.05,0.07]
,'max_bin': [64,128,256]
,'reg_alpha':[0,1,2,3,5,7]
,'reg_lambda':[0,1,2,3,5,7]
,'colsample_bytree':[0.03,0.05,0.6,0.7,0.8,0.9]
,'min_child_weight':[0.1,0.25,0.5,0.8,1,1.5,2,3]
}
mdl = lgb.LGBMRegressor(boosting_type='dart',
objective='regression',
n_jobs=-1,
silent=True,
max_depth=params['max_depth'],
max_bin=params['max_bin'],
subsample_for_bin= params['subsample_for_bin'],
subsample=params['subsample'],
subsample_freq=params['subsample_freq'],
min_split_gain=params['min_split_gain'],
min_child_weight=params['min_child_weight'],
min_child_samples=params['min_child_samples'], 
)
 mdl.get_params().keys()
grid=RandomizedSearchCV(mdl, gridParams, scoring=make_scorer(score_func=r2_score, greater_is_better=True)
 ,n_iter=25,n_jobs=-1)
 grid.fit(X_trainD,y_trainD)
 print(grid.best_params_)
 print(grid.best_score_)
 params['min_split_gain']= grid.best_params_['min_split_gain']
params['learning_rate']= grid.best_params_['learning_rate']
params['max_bin']= grid.best_params_['max_bin']
params['num_leaves']= grid.best_params_['num_leaves']
params['drop_rate']= grid.best_params_['drop_rate']
params['reg_alpha']= grid.best_params_['reg_alpha']
params['reg_lambda']= grid.best_params_['reg_lambda']
params['colsample_bytree']= grid.best_params_['colsample_bytree']
params['min_child_weight']= grid.best_params_['min_child_weight']
train_data=lgb.Dataset(X_trainD,label=y_trainD)
 lgbm_cases= lgb.train(params,
train_data,
600,
verbose_eval=4)
 train_pool_d = Pool(X_trainD,
label=y_trainD 
)
 val_pool_d = Pool(X_testD,
label=y_testD 
)
 test_pool_d = Pool(dfteste_d,
label=resposta_d 
)
 model = CatBoostRegressor(objective='RMSE')
 model.fit(train_pool_d, plot=True, eval_set=val_pool_d, verbose=500 )<compute_train_metric>
train_set = pd.read_csv(".. /input/train.csv")
 train_dfY = train_set['Survived']
test_set = pd.read_csv(".. /input/test.csv")
 submission = test_set['PassengerId'].copy()
print(train_set.shape)
 print(test_set.shape)
 print(train_dfY.shape)
resp=model.predict(test_pool_d)
 respLGB=lgbm_cases.predict(dfteste_d)
 print("MSE CatBoost: %.4f" %mean_squared_error(resp,resposta))
print("MSE GradientBoosting: %.4f" %mean_squared_error(rGB,resposta))
print("MSE RandomForest: %.4f" %mean_squared_error(rRF,resposta))
print("MSE LightGBM: %.4f" %mean_squared_error(respLGB,resposta))
dftr_d['Previsto']=resp
dftr_d['Resposta']=resposta_d
dftr_d[dftr_d['Local'].isin(['Brazil','US/New York','US/New Jersey','US/Florida','Italy','Spain','France','Germany'])]<feature_engineering>
train_set['Cabin'].value_counts()
dftr_d['Previsto']=np.where(( dftr_d['Crescimento_2'])>(dftr_d['Crescimento_1']),dftr_d['Previsto'],dftr_d['Previsto']/7)
 copy_dftr=dftr_d.copy()
dftr_d=df_f.pivot_table(index='Local',columns='Date',values='Mortalidade' ).reset_index()
dftr_copy=dftr_d.copy()
C1=np.where(
(dftr_d.iloc[: , -15].values)==0, 
(np.power(dftr_d.iloc[: , -8].values/(( dftr_d.iloc[: , -15].values)+1),1/7)) -(1)
 ,(np.power(dftr_d.iloc[: , -8].values/(( dftr_d.iloc[: , -15].values)) ,1/7)) -(1)
)
 C1=np.where(C1<0,0,C1)
 C2=np.where(
(dftr_d.iloc[: , -8].values)==0, 
(np.power(dftr_d.iloc[: , -1].values/(( dftr_d.iloc[: , -8].values)+1),1/7)) -(1)
 ,(np.power(dftr_d.iloc[: , -1].values/(( dftr_d.iloc[: , -8].values)) ,1/7)) -(1)
)
 C2=np.where(C2<0,0,C2)
 dftr_d['Crescimento_1']=C1
dftr_d['Crescimento_2']=C2
Beta0=0.25
Beta1=copy_dftr['Previsto']
dftr_d['Cres_2020-04-15']=dftr_d['Crescimento_2']*(( dftr_d['Crescimento_2']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-16']=dftr_d['Cres_2020-04-15']*(( dftr_d['Cres_2020-04-15']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-17']=dftr_d['Cres_2020-04-16']*(( dftr_d['Cres_2020-04-16']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-18']=dftr_d['Cres_2020-04-17']*(( dftr_d['Cres_2020-04-17']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-19']=dftr_d['Cres_2020-04-18']*(( dftr_d['Cres_2020-04-18']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-20']=dftr_d['Cres_2020-04-19']*(( dftr_d['Cres_2020-04-19']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-21']=dftr_d['Cres_2020-04-20']*(( dftr_d['Cres_2020-04-20']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-22']=dftr_d['Cres_2020-04-21']*(( dftr_d['Cres_2020-04-21']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-23']=dftr_d['Cres_2020-04-22']*(( dftr_d['Cres_2020-04-22']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-24']=dftr_d['Cres_2020-04-23']*(( dftr_d['Cres_2020-04-23']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-25']=dftr_d['Cres_2020-04-24']*(( dftr_d['Cres_2020-04-24']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-26']=dftr_d['Cres_2020-04-25']*(( dftr_d['Cres_2020-04-25']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-27']=dftr_d['Cres_2020-04-26']*(( dftr_d['Cres_2020-04-26']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-28']=dftr_d['Cres_2020-04-27']*(( dftr_d['Cres_2020-04-27']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-29']=dftr_d['Cres_2020-04-28']*(( dftr_d['Cres_2020-04-28']*(Beta1)+Beta0))
dftr_d['Cres_2020-04-30']=dftr_d['Cres_2020-04-29']*(( dftr_d['Cres_2020-04-29']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-01']=dftr_d['Cres_2020-04-30']*(( dftr_d['Cres_2020-04-30']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-02']=dftr_d['Cres_2020-05-01']*(( dftr_d['Cres_2020-05-01']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-03']=dftr_d['Cres_2020-05-02']*(( dftr_d['Cres_2020-05-02']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-04']=dftr_d['Cres_2020-05-03']*(( dftr_d['Cres_2020-05-03']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-05']=dftr_d['Cres_2020-05-04']*(( dftr_d['Cres_2020-05-04']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-06']=dftr_d['Cres_2020-05-05']*(( dftr_d['Cres_2020-05-05']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-07']=dftr_d['Cres_2020-05-06']*(( dftr_d['Cres_2020-05-06']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-08']=dftr_d['Cres_2020-05-07']*(( dftr_d['Cres_2020-05-07']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-09']=dftr_d['Cres_2020-05-08']*(( dftr_d['Cres_2020-05-08']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-10']=dftr_d['Cres_2020-05-09']*(( dftr_d['Cres_2020-05-09']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-11']=dftr_d['Cres_2020-05-10']*(( dftr_d['Cres_2020-05-10']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-12']=dftr_d['Cres_2020-05-11']*(( dftr_d['Cres_2020-05-11']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-13']=dftr_d['Cres_2020-05-12']*(( dftr_d['Cres_2020-05-12']*(Beta1)+Beta0))
dftr_d['Cres_2020-05-14']=dftr_d['Cres_2020-05-13']*(( dftr_d['Cres_2020-05-13']*(Beta1)+Beta0))<prepare_output>
datasets = [train_set, test_set]
originalData = train_set
for dataset in datasets:
dataset['Age'].fillna(dataset["Age"].median() , inplace=True)
 dataset['Embarked'].fillna(dataset["Embarked"].mode() [0], inplace=True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 dataset.drop(['PassengerId','Cabin', 'Ticket'], axis=1, inplace=True)
 dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
print('Ahora columnas null en el trainset')
 print(train_set.isnull().sum())
 print('Ahora columnas null en el test set')
 print(test_set.isnull().sum())
dftr_d['2020-04-15']=(1+dftr_d['Cres_2020-04-15'])*dftr_d['2020-04-14']
dftr_d['2020-04-16']=(1+dftr_d['Cres_2020-04-16'])*dftr_d['2020-04-15']
dftr_d['2020-04-17']=(1+dftr_d['Cres_2020-04-17'])*dftr_d['2020-04-16']
dftr_d['2020-04-18']=(1+dftr_d['Cres_2020-04-18'])*dftr_d['2020-04-17']
dftr_d['2020-04-19']=(1+dftr_d['Cres_2020-04-19'])*dftr_d['2020-04-18']
dftr_d['2020-04-20']=(1+dftr_d['Cres_2020-04-20'])*dftr_d['2020-04-19']
dftr_d['2020-04-21']=(1+dftr_d['Cres_2020-04-21'])*dftr_d['2020-04-20']
dftr_d['2020-04-22']=(1+dftr_d['Cres_2020-04-22'])*dftr_d['2020-04-21']
dftr_d['2020-04-23']=(1+dftr_d['Cres_2020-04-23'])*dftr_d['2020-04-22']
dftr_d['2020-04-24']=(1+dftr_d['Cres_2020-04-24'])*dftr_d['2020-04-23']
dftr_d['2020-04-25']=(1+dftr_d['Cres_2020-04-25'])*dftr_d['2020-04-24']
dftr_d['2020-04-26']=(1+dftr_d['Cres_2020-04-26'])*dftr_d['2020-04-25']
dftr_d['2020-04-27']=(1+dftr_d['Cres_2020-04-27'])*dftr_d['2020-04-26']
dftr_d['2020-04-28']=(1+dftr_d['Cres_2020-04-28'])*dftr_d['2020-04-27']
dftr_d['2020-04-29']=(1+dftr_d['Cres_2020-04-29'])*dftr_d['2020-04-28']
dftr_d['2020-04-30']=(1+dftr_d['Cres_2020-04-30'])*dftr_d['2020-04-29']
dftr_d['2020-05-01']=(1+dftr_d['Cres_2020-05-01'])*dftr_d['2020-04-30']
dftr_d['2020-05-02']=(1+dftr_d['Cres_2020-05-02'])*dftr_d['2020-05-01']
dftr_d['2020-05-03']=(1+dftr_d['Cres_2020-05-03'])*dftr_d['2020-05-02']
dftr_d['2020-05-04']=(1+dftr_d['Cres_2020-05-04'])*dftr_d['2020-05-03']
dftr_d['2020-05-05']=(1+dftr_d['Cres_2020-05-05'])*dftr_d['2020-05-04']
dftr_d['2020-05-06']=(1+dftr_d['Cres_2020-05-06'])*dftr_d['2020-05-05']
dftr_d['2020-05-07']=(1+dftr_d['Cres_2020-05-07'])*dftr_d['2020-05-06']
dftr_d['2020-05-08']=(1+dftr_d['Cres_2020-05-08'])*dftr_d['2020-05-07']
dftr_d['2020-05-09']=(1+dftr_d['Cres_2020-05-09'])*dftr_d['2020-05-08']
dftr_d['2020-05-10']=(1+dftr_d['Cres_2020-05-10'])*dftr_d['2020-05-09']
dftr_d['2020-05-11']=(1+dftr_d['Cres_2020-05-11'])*dftr_d['2020-05-10']
dftr_d['2020-05-12']=(1+dftr_d['Cres_2020-05-12'])*dftr_d['2020-05-11']
dftr_d['2020-05-13']=(1+dftr_d['Cres_2020-05-13'])*dftr_d['2020-05-12']
dftr_d['2020-05-14']=(1+dftr_d['Cres_2020-05-14'])*dftr_d['2020-05-13']<filter>
for dataset in datasets:
StringArray = dataset['Name'].str.split(", ", expand=True)
 StringArray = StringArray[1].str.split(".", expand=True)
 dataset['Title'] = StringArray[0]
title_names = dataset['Title'].value_counts()
print('===Prior to grouping===')
 print(title_names)
 title_names =(dataset['Title'].value_counts() > 7)
 dataset['Title'] = dataset['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == False else x)
 print('===Post grouping===')
 print(dataset['Title'].value_counts())
dftr_d[dftr_d['Local'].isin(['Brazil','US/New York','US/New Jersey','US/Florida','Italy','Spain','France','Germany'])]<feature_engineering>
label = LabelEncoder()
enc = OneHotEncoder(handle_unknown='ignore')
 train_set = datasets[0]
test_set = datasets[1]
train_set = train_set.drop(['Survived'], axis=1)
 EncodedDataFrames = []
print(train_set.shape, test_set.shape)
 wholeData = pd.concat([train_set, test_set], ignore_index=True)
 wholeData['Sex_Code'] = label.fit_transform(wholeData['Sex'])
 wholeData['Embarked_Code'] = label.fit_transform(wholeData['Embarked'])
 wholeData['Title_Code'] = label.fit_transform(wholeData['Title'])
 sex_encoded = to_categorical(wholeData['Sex_Code'])
 SexOneHot = pd.DataFrame(sex_encoded, columns = ["Female", "Male"])
 embarked_encoded = to_categorical(wholeData['Embarked_Code'])
 EmbarkedOneHot = pd.DataFrame(embarked_encoded, columns = ["C", "Q", "S"])
 title_encoded = to_categorical(wholeData['Title_Code'])
 TitleOneHot = pd.DataFrame(title_encoded, columns = ["Master", "Misc", "Miss", "Mr", "Mrs"])
 pclass_encoded = to_categorical(wholeData['Pclass'])
 PClassOneHot = EmbarkedOneHot = pd.DataFrame(pclass_encoded, columns = ["0th Class", "1th Class", "2nd Class", "3rd Class"])
 PClassOneHot = PClassOneHot.drop("0th Class", axis=1)
 wholeData = pd.concat([wholeData, SexOneHot, EmbarkedOneHot, TitleOneHot, PClassOneHot], axis=1)
 EncodedDataFrames.append(wholeData)
 print(wholeData.shape)
dfm=df_f.pivot_table(index='Local',columns='Date',values='Fatalities' ).reset_index()
mortes_adj=dfm.iloc[: , -1].values.sum() / dft_copy.iloc[: , -1].values.sum()
dft['mortes']=dfm.iloc[: , -1].values / dft_copy.iloc[: , -1].values
print(mortes_adj)
 dft.head()<filter>
wholeData = wholeData.drop(['Name', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title', 'Pclass'], axis= 1)
 [train_set, test_set] = np.split(wholeData, [891], axis= 0)
 train_dfX = train_set
print(train_dfX.shape, train_dfY.shape)
 train_dfX,val_dfX,train_dfY, val_dfY = train_test_split(train_dfX,train_dfY , test_size=0.1, stratify=train_dfY)
 print("Entrenamiento: ",train_dfX.shape, train_dfY.shape)
 print("Validacion : ",val_dfX.shape, val_dfY.shape)
 print("Test: ", test_set.shape)
dft[dft['Local'].isin(['Brazil','US/New York','US/New Jersey','US/Florida','Italy','Spain','France','Germany'])]<feature_engineering>

 model = Sequential()
model.add(Dense(40, kernel_initializer = 'glorot_normal', bias_initializer='zeros', activation = 'relu', kernel_regularizer=regularizers.l2(0.01), input_dim = 19))
model.add(Dropout(0.2))
model.add(Dense(20, kernel_initializer = 'glorot_normal', bias_initializer='zeros', activation = 'relu', kernel_regularizer=regularizers.l2(0.01)))
 model.add(Dropout(0.2))
model.add(Dense(1, kernel_initializer = 'glorot_normal', bias_initializer='zeros', activation = 'sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy'])
 model.summary()
dft['mortes']=np.where(dft['mortes']>(2*mortes_adj),(2*mortes_adj),np.where(dft['mortes']<(mortes_adj/2),(mortes_adj/2),dft['mortes']))<merge>
train_history = model.fit(train_dfX, train_dfY, batch_size=24, epochs= 20, validation_data=(val_dfX, val_dfY))
<merge><EOS> 
y_test = model.predict(test_set)
 y_formatted = np.where(y_test > 0.5, 1, 0)
 y_dataFrame = pd.DataFrame(np.ravel(y_formatted), columns=['Survived'])
 submission = pd.concat([submission, y_dataFrame], axis=1)
 submission.to_csv('submission.csv', index=False )