Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
seed_value=2509
os.environ['PYTHONHASHSEED']=str(seed_value)
 np.random.seed(seed_value)
 rn.seed(seed_value)
submission.target =(submission.target + pd.read_csv("./dnn_cv_submission.csv" ).target)/ 2
submission.to_csv("./blend_submission.csv", index=False )<load_from_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv', index_col='id' )<prepare_output>
combo = [train, test]
for item in combo:
item['Sex'] = item['Sex'].map({'male': 1, 'female': 0} ).astype(int )
def summary(df):
summary = pd.DataFrame(df.dtypes, columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name', 'dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
return summary
summary(train )<feature_engineering>
for item in combo:
item['FamilySize'] = item['SibSp'] + item['Parch']+1
train['missing_count'] = train.isnull().sum(axis=1)
 test['missing_count'] = test.isnull().sum(axis=1 )<define_variables>
train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean()
missing_number = -99999
missing_string = 'MISSING_STRING'<define_variables>
for item in combo:
item.loc[(item['FamilySize'] ==1), 'Fsize'] = 0
item.loc[(item['FamilySize'] ==2), 'Fsize'] = 1
item.loc[(item['FamilySize'] ==3), 'Fsize'] = 2
item.loc[(item['FamilySize'] ==4), 'Fsize'] = 3
item.loc[(item['FamilySize'] >4), 'Fsize'] = 4
item['Fsize']=item['Fsize'].astype(int )
numerical_features = [
'bin_0', 'bin_1', 'bin_2',
'ord_0',
'day', 'month'
]
string_features = [
'bin_3', 'bin_4',
'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5',
'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'
]<categorify>
for item in combo:
item['Title'] = item.Name.str.extract('([A-Za-z]+)\.', expand=False)
 item = item.drop(['Name'], axis=1, inplace=True)
 print("Titles in test dataset")
 print(' ')
 print(test.Title.unique())
 print(' ')
 print("Titles in train dataset")
 print(' ')
 print(train.Title.unique() )
def impute(train, test, columns, value):
for column in columns:
train[column] = train[column].fillna(value)
 test[column] = test[column].fillna(value )<compute_test_metric>
for item in combo:
item['Title'] = item['Title'].replace(['Ms','Lady', 'Countess','Dona'],'Mrs')
 item['Title'] = item['Title'].replace(['Mme','Mlle'], 'Miss')
 item['Title'] = item['Title'].replace(['Major', 'Sir', 'Jonkheer', 'Dr','Col','Don', 'Capt','Rev'],'Mr' )
impute(train, test, numerical_features, missing_number)
 impute(train, test, string_features, missing_string )<drop_column>
print(train.loc[(train.PassengerId== 797)] )
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train.loc[train['ord_5'] == missing_string, 'ord_5_1'] = missing_string
train.loc[train['ord_5'] == missing_string, 'ord_5_2'] = missing_string
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test.loc[test['ord_5'] == missing_string, 'ord_5_1'] = missing_string
test.loc[test['ord_5'] == missing_string, 'ord_5_2'] = missing_string
test = test.drop('ord_5', axis=1 )<define_variables>
print("
print(train.Title.value_counts())
 print("
print(test.Title.value_counts() )
simple_features = [
'missing_count'
]
oe_features = [
'bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4',
'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',
'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5_1', 'ord_5_2',
'day', 'month'
]
ohe_features = oe_features
target_features = [
'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'
]<prepare_x_and_y>
train.isnull().sum()
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test<normalization>
train.isnull().sum()
scaler = StandardScaler()
simple_x_train = scaler.fit_transform(x_train[simple_features])
 simple_x_test = scaler.transform(x_test[simple_features] )<categorify>
test.isnull().sum()
ohe = OneHotEncoder(dtype='uint16', handle_unknown="ignore")
 ohe_x_train = ohe.fit_transform(x_train[ohe_features])
 ohe_x_test = ohe.transform(x_test[ohe_features] )<categorify>
test.isnull().sum()
oe = OrdinalEncoder()
oe_x_train = oe.fit_transform(x_train[oe_features])
 oe_x_test = oe.transform(x_test[oe_features] )<import_modules>
for item in combo:
item['Cabin'].fillna('N', inplace=True)
 item['Cabin'] = item['Cabin'].map(lambda c: c[0] )
from category_encoders import TargetEncoder
from sklearn.model_selection import StratifiedKFold<categorify>
for item in combo:
item['Cabin'] = item['Cabin'].replace(['A', 'B','C',"D",'E','T'], 0)
 item['Cabin'] = item['Cabin'].replace(['F','G'], 0)
 item['Cabin'] = item['Cabin'].replace(['N'], 1 )
def transform(transformer, x_train, y_train, cv):
oof = pd.DataFrame(index=x_train.index, columns=x_train.columns)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train.loc[train_idx]
y_train_train = y_train.loc[train_idx]
x_train_valid = x_train.loc[valid_idx]
transformer.fit(x_train_train, y_train_train)
 oof_part = transformer.transform(x_train_valid)
 oof.loc[valid_idx] = oof_part
return oof<categorify>
train[['Cabin','Survived']].groupby(['Cabin'], as_index=False ).mean()
target = TargetEncoder(drop_invariant=True, smoothing=0.2)
 cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 target_x_train = transform(target, x_train[target_features], y_train, cv ).astype('float')
 target.fit(x_train[target_features], y_train)
 target_x_test = target.transform(x_test[target_features] ).astype('float' )<concatenate>
train['Embarked'] = train['Embarked'].fillna('S')
 combo = [train, test]
for item in combo:
item['Embarked'] = item['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
x_train = scipy.sparse.hstack([ohe_x_train, simple_x_train, target_x_train] ).tocsr()
x_test = scipy.sparse.hstack([ohe_x_test, simple_x_test, target_x_test] ).tocsr()<train_model>
combo = [train, test]
for item in combo:
item['Age'] = item.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.mean()))
logit = LogisticRegression(C=0.54321, solver='lbfgs', max_iter=10000)
 logit.fit(x_train, y_train)
 y_pred_logit = logit.predict_proba(x_test)[:, 1]<concatenate>
test['Fare'].fillna(test['Fare'].median() , inplace = True )
x_train = np.concatenate(( oe_x_train, simple_x_train, target_x_train), axis=1)
 x_test = np.concatenate(( oe_x_test, simple_x_test, target_x_test), axis=1 )<define_variables>
for item in combo:
item.loc[ item['Fare'] <= 7.55, 'Fare'] = 0
item.loc[(item['Fare'] > 7.55)&(item['Fare'] <= 10.5), 'Fare'] = 1
item.loc[(item['Fare'] > 10.5)&(item['Fare'] <= 14.454), 'Fare'] = 2
item.loc[(item['Fare'] > 14.454)&(item['Fare'] <= 21.67), 'Fare'] = 3
item.loc[(item['Fare'] > 21.67)&(item['Fare'] <= 77.958), 'Fare'] = 4
item.loc[ item['Fare'] > 77.958, 'Fare'] = 5
item['Fare'] = item['Fare'].astype(int )
categorial_part = oe_x_train.shape[1]<import_modules>
train[['Pclass','Survived']].groupby(['Pclass'], as_index=False ).mean()
import tensorflow as tf<compute_test_metric>
train[['Fare','Survived']].groupby(['Fare'], as_index=False ).mean()
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<choose_model_class>
for item in combo:
item = item.drop(['Ticket'], axis=1, inplace=True )
def make_model(data, categorial_part):
inputs = []
categorial_outputs = []
for idx in range(categorial_part):
n_unique = np.unique(data[:,idx] ).shape[0]
n_embeddings = int(min(np.ceil(n_unique / 2), 50))
inp = tf.keras.layers.Input(shape=(1,))
inputs.append(inp)
 x = tf.keras.layers.Embedding(n_unique + 1, n_embeddings )(inp)
 x = tf.keras.layers.SpatialDropout1D(0.3 )(x)
 x = tf.keras.layers.Reshape(( n_embeddings,))(x)
 categorial_outputs.append(x)
 x1 = tf.keras.layers.Concatenate()(categorial_outputs)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 inp = tf.keras.layers.Input(shape=(data.shape[1] - categorial_part,))
inputs.append(inp)
 x2 = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Concatenate()([x1, x2])
 x = tf.keras.layers.Dense(128, activation='relu' )(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 y = tf.keras.layers.Dense(1, activation='sigmoid', name='dense_output' )(x)
 print('Expected number of inputs:', len(inputs))
model = tf.keras.Model(inputs=inputs, outputs=y)
 model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=3e-5), metrics=['accuracy', auc])
 return model<define_variables>
train['Age2']=-1
test['Age2']=-1
combo=[train,test]
for item in combo:
item.loc[(item['Age'] <=4), 'Age2'] =0
item.loc[(item['Age']<=17)&(item['Age'] >4), 'Age2'] =1
item.loc[(item['Age'] >=18)&(item['Age'] <=32), 'Age2'] =2
item.loc[(item['Age'] >32)&(item['Age'] <=42), 'Age2']= 3
item.loc[(item['Age'] >42)&(item['Age'] <=52), 'Age2']= 4
item.loc[(item['Age'] >52)&(item['Age'] <=60), 'Age2']= 5
item.loc[(item['Age'] >60), 'Age2']= 6
def make_inputs(data, categorial_part):
inputs = []
for idx in range(categorial_part):
inputs.append(data[:, idx])
 inputs.append(data[:, categorial_part:])
 return inputs<split>
train[['Survived','Age2']].groupby(['Age2'], as_index=False ).mean()
n_splits = 50
trained_estimators = []
histories = []
scores = []
cv = KFold(n_splits=n_splits, random_state=42)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
K.clear_session()
estimator = make_model(x_train, categorial_part)
 es = tf.keras.callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=10,
verbose=1, mode='max', restore_best_weights=True)
 rl = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=3, min_lr=1e-6, mode='max', verbose=1)
 history = estimator.fit(make_inputs(x_train_train, categorial_part), y_train_train, batch_size=1024, epochs=100, callbacks=[es, rl],
validation_data=(make_inputs(x_train_valid, categorial_part), y_train_valid))
trained_estimators.append(estimator)
 histories.append(history)
 oof_part = estimator.predict(make_inputs(x_train_valid, categorial_part))
score = roc_auc_score(y_train_valid, oof_part)
 print('Fold score:', score)
 scores.append(score )<compute_test_metric>
pearson_coef1 , p_value1 =scipy.stats.pearsonr(train["Cabin"],train["Pclass"])
 print("Cabin - Pclass", pearson_coef1," Correlation certainty =",p_value1 )
print('Mean score:', np.mean(scores))<predict_on_test>
for item in combo:
item = item.drop(['Sex'], axis=1, inplace=True )
y_pred = np.zeros(x_test.shape[0])
 x_test_inputs = make_inputs(x_test, categorial_part)
 for estimator in trained_estimators:
y_pred += estimator.predict(x_test_inputs ).reshape(-1)/ len(trained_estimators )<prepare_output>
for item in combo:
item.loc[(( item['Title']=='Miss')&(item['Age'] <=4)) , 'Title'] = 'child'
item.loc[(( item['Title']=='Master')&(item['Age'] <=4)) , 'Title'] = 'child'
item.loc[(( item['Title']=='Miss')&(item['Age'] <=17)) , 'Title'] = 'kid'
item.loc[(( item['Title']=='Master')&(item['Age'] <=17)) , 'Title'] = 'kid'
item.loc[(( item['Title']=='Master')&(item['Age'] >17)) , 'Title'] = 'Mr'
item.loc[(( item['Title']=='Mrs')&(item['Parch']!=0)) ,'Title']='MrsYesCh'
item.loc[(( item['Title']=='Mrs')&(item['Parch']==0)) ,'Title']='MrsNoCh'
item = item.drop(['SibSp','Parch'], axis=1, inplace=True )
y_pred_keras = y_pred<compute_test_metric>
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
y_pred = np.add(y_pred_logit, y_pred_keras)/ 2<save_to_csv>
train['Title'] = train['Title'].map({ 'Mr': 0 ,'Miss':1, 'MrsNoCh':2 ,'MrsYesCh':3,'kid':4,'child':5} ).astype(int)
 test['Title'] = test['Title'].map({'Mr': 0 ,'Miss':1, 'MrsNoCh':2 ,'MrsYesCh':3,'kid':4, 'child':5} ).astype(int )
submission = pd.read_csv('.. /input/cat-in-the-dat-ii/sample_submission.csv', index_col='id')
 submission['target'] = y_pred
submission.to_csv('logit_keras.csv' )<install_modules>
for item in combo:
item = item.drop(['Age'], axis=1, inplace=True )
!pip install --no-warn-conflicts -q deepctr<import_modules>
X_train = train.drop(['Survived','PassengerId','FamilySize','FareBin'], axis=1)
 col=(X_train.columns)
 Y_train = train["Survived"]
X_test=test.drop(["PassengerId","FamilySize"], axis=1 )
warnings.simplefilter('ignore' )<load_from_csv>
scaler = StandardScaler()
scaler.fit_transform(X_train)
 scaler.transform(X_test )
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )<feature_engineering>
test1 = SelectKBest(score_func= f_classif, k=4)
 fit = test1.fit(X_train, Y_train)
 set_printoptions(precision=3)
 print(col)
 print(' ',fit.scores_ )
test['target'] = -1
test.head()<concatenate>
model = ExtraTreesClassifier(n_estimators=10)
 model.fit(X_train, Y_train)
 print(col)
 print('Feature importance')
 print(model.feature_importances_ )
data = pd.concat([train, test] ).reset_index(drop=True )<feature_engineering>
rfe = RFE(estimator=DecisionTreeClassifier() , n_features_to_select=4)
 fit = rfe.fit(X_train, Y_train)
 print('Features : ',col)
 print("Num Features: ",fit.n_features_)
 print("Selected Features: ",fit.support_)
 print("Feature Ranking: ",fit.ranking_ )
data['null'] = data.isna().sum(axis=1 )<feature_engineering>
X_train = pd.get_dummies(X_train, columns = ["Title"],prefix="N")
 X_train = pd.get_dummies(X_train, columns = ["Fare"],prefix="F",drop_first=True)
 X_test = pd.get_dummies(X_test, columns = ["Title"],prefix="N")
 X_test = pd.get_dummies(X_test, columns = ["Fare"],prefix="F",drop_first=True)
 Y_train = train["Survived"].values
sparse_features = [feat for feat in train.columns if feat not in ['id','target']]
data[sparse_features] = data[sparse_features].fillna('-1', )<categorify>
X_train.drop(["Embarked","Cabin",'Age2'], axis=1, inplace=True)
 X_test.drop(["Embarked","Cabin",'Age2'], axis=1,inplace=True)
 print("Training set X: ", X_train.shape, "Training set Y:",Y_train.shape, "Testing set X", X_test.shape )
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat].fillna('-1' ).astype(str ).values )<prepare_x_and_y>
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier,AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
import lightgbm as lgb
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from sklearn.ensemble import VotingClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression
import keras
from keras import models,layers,optimizers
from keras.optimizers import Adam
from keras.layers import Activation, Dense ,Dropout
from keras.callbacks import EarlyStopping
from numpy import mean , std
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
from numpy import arange, argmax
from sklearn import metrics
from sklearn.metrics import auc,confusion_matrix,classification_report,roc_curve,accuracy_score,roc_auc_score
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True )<count_unique_values>
kfld = StratifiedKFold(n_splits=5, shuffle=True, random_state=2509)
 GB_param = {"n_estimators":[100,150,250,500], "learning_rate": [0.001,0.01,0.05,0.1,0.2],
"min_samples_split":[3,10,31],"max_depth":[3,5,7], "max_features":[4,7,8,9,10,11,13],
"subsample":[0.8,1], "criterion": ["mae"]}
AdaB_param ={'n_estimators': [10,20,50,100,200,500],'learning_rate':[0.001,0.01,0.03,0.05,0.07,0.1,0.2,0.3] }
SVC_param = { 'kernel' : ['rbf', 'linear', 'poly', 'sigmoid'],
'gamma': ['auto'],
'C': [0.01,0.1,0.2,0.5,1,2,3,4,5,6,7,8,9,10,11,12,20,50,100],
'shrinking' : [True, False]}
RF_param = {'n_estimators' : [20,50,70,100,500], 'max_features' : [3,4,5,6,7,9,10,13],
'max_depth' : [3,5,7],'criterion' : ['gini'], 'min_samples_split' : [4,10,31,64]}
XGB_param= {"learning_rate" :[0.001,0.01,0.05,0.1,0.2], "max_depth":[3,5,7],
"gamma": [0,0.5,1],"subsample":[0.8,1], "alpha" : [0,0.5,1], "n_estimators" : [100,500,800],
'colsample_bytree': [0.6,0.8,1], "objective": ['binary:logistic']}
LGparam= {'boosting_type': ['gbdt'],'num_leaves': [4,10,31],'subsample': [0.8,1],
'colsample_bytree': [0.6,0.8,1],'reg_alpha': [0,0.5,1],'reg_lambda':[0,0.5,1],
'max_depth':[3,5,7],'n_estimators' :[100,500,800],'learning_rate': [0.001,0.005,0.01,0.05,0.1],
'objective': ['binary'],'metric': ['binary_logloss']}
MPL_param = {'hidden_layer_sizes': [(16,8,1),(32,16,1),(32,32)],
'activation': ['tanh', 'relu'],'solver': ['adam'], 'alpha': [0.01, 0.05]}
Knn_param = dict(leaf_size=list(range(1,30)) , n_neighbors=list(range(1,50)) , weights = ['uniform','distance'],p=[1,2])
 clfxgb =XGBClassifier(random_state=seed_value)
 clfgb = GradientBoostingClassifier(random_state=seed_value)
 clfAdaB=AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=5),random_state=seed_value)
 clfsvc = SVC(random_state=seed_value)
 clfRf=RandomForestClassifier(random_state=seed_value)
 clfxgb =XGBClassifier(random_state=seed_value)
 clfknn=KNeighborsClassifier()
clfLGB = LGBMClassifier(seed=seed_value)
 clfmlp = MLPClassifier(max_iter=200)
 classfrs={clfxgb:XGB_param,clfknn:Knn_param,clfLGB:LGparam,clfRf:RF_param,clfsvc:SVC_param,
clfmlp:MPL_param,clfAdaB:AdaB_param, clfgb:GB_param}
fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique())for feat in sparse_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns )<compute_test_metric>
randomforest = RandomForestClassifier(criterion = 'gini',bootstrap= True, max_depth= 5, max_features=10, min_samples_split= 31, n_estimators=20,random_state=seed_value)
 svc_model = SVC(C=0.5, cache_size=200,
decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf', max_iter=-1, probability=True, random_state=seed_value, shrinking=True, tol=0.001,verbose=False)
 knn = KNeighborsClassifier(leaf_size= 6,n_neighbors = 16, p=1 ,weights = 'uniform')
 modellt = LGBMClassifier(boosting_type='gbdt',num_leaves = 31,max_depth =5,learning_rate =0.05,subsample= 1,colsample_bytree = 0.8, reg_lambda =0.5,reg_alpha= 0.5,
n_estimators=500,objective='binary',metric='binary_logloss', seed=seed_value)
 xgb_model=XGBClassifier(alpha =0.5, colsample_bytree =0.8 ,gamma=0.5,learning_rate= 0.1, max_depth= 5,n_estimators=100,
objective= 'binary:logistic',subsample= 0.8, random_state=seed_value)
 modelM=MLPClassifier(hidden_layer_sizes=(16,8,1),activation='tanh',solver='adam', alpha=0.01, batch_size=32, learning_rate='adaptive', learning_rate_init=0.001, max_iter=850, shuffle=False, random_state=seed_value, tol=0.001, verbose=False,early_stopping=False,validation_fraction=0.3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 AdaB=AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=5),learning_rate=0.001,n_estimators=100,random_state=seed_value)
 gb = GradientBoostingClassifier(criterion='mae',n_estimators=150,learning_rate = 0.1,max_features=7,max_depth =3,
min_samples_leaf=3, min_samples_split= 32, subsample= 1,random_state=seed_value)
 NN1=('NN', modelM)
 XG1=('XGboost', xgb_model)
 GR1=('GradientB',gb)
 SVM1=('SVM',svc_model)
 ADA1=('Ada',AdaB)
 KNN1=('KNN',knn)
 RF1=('RF',randomforest)
 LG1=('LightGBM',modellt )
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<compute_train_metric>
def evaluate_model(model):
seed=7
kfld = StratifiedKFold(n_splits=5,shuffle=True,random_state=seed)
 scores = cross_val_score(model, X_train , Y_train, scoring='accuracy', cv=kfld, n_jobs=-1, error_score='raise')
 return scores
clfs={'LightGBM':[modellt,78.707],'XGBoost':[xgb_model,99978.229],'GradientBoost':[gb,78.468]
,'AdaBoost':[AdaB,78.229],'RandomForest':[randomforest,78.707],
'SVM':[svc_model,78.229],'KNN':[knn,80.143],'MLP':[modelM,79.168]}
scores={}
for name, model in clfs.items() :
metr=evaluate_model(model[0])
 scores.update({name:[mean(metr),std(metr),model[1]]})
 results_table = pd.DataFrame(scores, index=['Mean_accuracy', 'Std', 'Leaderboard'] ).round(decimals=3)
 results_table
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(K.epsilon() +pt_1)) -K.mean(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0 + K.epsilon()))
return focal_loss_fixed
get_custom_objects().update({'focal_loss_fn': focal_loss() } )<feature_engineering>
x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.2,shuffle=True ,random_state=123 )
def custom_gelu(x):
return 0.5 * x *(1 + tf.tanh(tf.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'custom_gelu': Activation(custom_gelu)} )<choose_model_class>
def evaluation(name,clf):
model=clf.fit(x_train,y_train)
 y_prob = model.predict_proba(x_test)[:, 1]
prAuc = average_precision_score(y_test, y_prob)
 y_pred = model.predict(x_test)
 acc = balanced_accuracy_score(y_test, y_pred)
 f1= f1_score(y_test, y_pred)
 return prAuc,acc,f1
scores={}
for name, model in models_set.items() :
metr=evaluation(name,model)
 scores.update({name:[metr[0],metr[1],metr[2]]})
 results =(pd.DataFrame(scores, index=['PR-AUC','Balanced_Accuracy','F1 score'])).round(decimals=3)
 results
class WarmUpLearningRateScheduler(tf.keras.callbacks.Callback):

 def __init__(self, warmup_batches, init_lr, verbose=0):

 super(WarmUpLearningRateScheduler, self ).__init__()
self.warmup_batches = warmup_batches
self.init_lr = init_lr
self.verbose = verbose
self.batch_count = 0
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.batch_count = self.batch_count + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
if self.batch_count <= self.warmup_batches:
lr = self.batch_count*self.init_lr/self.warmup_batches
K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: WarmUpLearningRateScheduler setting learning '
'rate to %s.' %(self.batch_count + 1, lr))<choose_model_class>
def create_clf(E,t):
if t=='Voting':
vclf=VotingClassifier(E, voting='hard')
 else:
vclf=StackingClassifier(E, final_estimator= LogisticRegression())
 return vclf
E1=[KNN1,NN1,SVM1,RF1,GR1]
E2=[KNN1,SVM1,RF1,GR1]
E3=[KNN1,SVM1,GR1]
E4=[KNN1,SVM1]
ensembles_set={'KNN-NN-SVM-RF-GR':[E1,'79.186','78.947'],'KNN-SVM-RF-GR':[E2,'79.425','79.186 ']
,'KNN-SVM-GR':[E3,'79.425','79.186'],'KNN-SVM1':[E4,'80.382','78.468']}
scores1={}
scores2={}
for clf, item in ensembles_set.items() :
newclv=create_clf(item[0],'Voting')
 newclST=create_clf(item[0],'Stacking')
 kfoldevalv=evaluate_model(newclv)
 kfoldevalst=evaluate_model(newclST)
 scores1.update({clf:[mean(kfoldevalv),std(kfoldevalv),item[1]]})
 scores2.update({clf:[mean(kfoldevalst),std(kfoldevalst),item[2]]})
 results1 = pd.DataFrame(scores1, index=['Mean_accuracy', 'Std','Leaderboard'])
 results2 = pd.DataFrame(scores2, index=['Mean_accuracy', 'Std','Leaderboard'])
 print('VOTING CLASSIFIERS')
 display(results1)
 print('STACKING CLASSIFIERS')
 display(results2 )
<define_variables><EOS> 
E=[KNN1,SVM1]
vot=VotingClassifier(E, voting='hard')
 ensemble = vot.fit(X_train, Y_train)
 y_vote = ensemble.predict(X_test)
 submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": y_vote
})
 submission.to_csv('Titanic.csv', index=False)
 print("Your submission was successfully saved!")