Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
def create_submission(test, preds_test, file_name):
predictions = []
for pred in preds_test:
if pred == 1:
predictions.append(1)
 else:
predictions.append(0)
 submission = pd.concat([test_orig['PassengerId'], pd.Series(predictions ).astype("int")], axis=1)
 submission.columns = ['PassengerId', 'Survived']
submission['Survived'] = submission['Survived'].astype("int")
 submission.to_csv(file_name, index = False)
 def run_model(train, test, cv_grid_params,file_name = "submission.csv", regression = False, gauss_proc = False):
X_train = train.drop(['Survived'], axis = 1)
 y_train = train['Survived']
X_test = test.drop(['Survived'], axis = 1)
 y_test = test['Survived']
grid_search = GridSearchCV(**cv_grid_params)
 grid_search.fit(X_train,y_train)
 model = cv_grid_params['estimator']
model.set_params(**grid_search.best_params_)
 model.fit(X_train,y_train)
 cv_scores = cross_val_score(model, X_train, y_train, cv = 10)
 print("Cross validation scores:" + str(cv_scores))
print("Mean score: " + str(cv_scores.mean()))
test_preds = model.predict(X_test)
 if regression == False:
print("Train confusion matrix:")
 print(confusion_matrix(y_train.astype("int"), model.predict(X_train ).astype("int")))
 print("Predicting and creating submission.")
 create_submission(X_test, test_preds.astype("float"), file_name)
 else:
train_preds = model.predict(X_train)
 preds_test = []
preds_train = []
for i in range(train_preds.shape[0]):
if train_preds[i] < 0.5:
preds_train.append(0)
 else:
preds_train.append(1)
 print("Train confusion matrix:")
 print(confusion_matrix(y_train, np.array(preds_train)))
 for i in range(test_preds.shape[0]):
if test_preds[i] < 0.5:
preds_test.append(0)
 else:
preds_test.append(1)
 print("Predicting and creating ridge submission.")
 create_submission(X_test, preds_test, file_name)
 return test_preds, model
%%time
learner4.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner4,
every='improvement',
monitor='accuracy',
name='best_model_4')]
)<save_to_csv>
seed = 123
train_orig = pd.read_csv(".. /input/train.csv")
 test_orig = pd.read_csv(".. /input/test.csv")
 data = pd.concat([test_orig,train_orig], sort = False)
 data = data.reset_index()
class_weights = {
0:(train_orig['Survived'] == 0 ).sum() /train_orig.shape[0],
1:(train_orig['Survived'] == 1 ).sum() /train_orig.shape[0]
}
data.describe()
learner4.load('best_model_4')
 preds, ids = learner4.get_preds(DatasetType.Test)
 y = torch.argmax(torch.exp(preds), dim=1)
 submission_4 = pd.DataFrame({ 'id': ids,'label': y })
 submission_4.to_csv("submission_4.csv", index=False )<normalization>
data.isna().sum()
flatten = lambda l: [np.float32(item)for sublist in l for item in sublist]
metrics_list_1 = flatten(learner1.recorder.metrics)
 metrics_list_2 = flatten(learner2.recorder.metrics)
 metrics_list_3 = flatten(learner3.recorder.metrics)
 metrics_list_4 = flatten(learner4.recorder.metrics )<save_to_csv>
data['Fare'] = data['Fare'].fillna(data['Fare'].dropna().median() )
blended_preds = np.round(( submission_1['label'] + submission_2['label'] +
submission_3['label'] + submission_4['label'])/4)
 blended_submission = pd.DataFrame({'id': ids, 'label': blended_preds})
 blended_submission['label'] = blended_submission['label'].astype(np.uint8)
 blended_submission.to_csv("blended_submission.csv", index=False )<import_modules>
print(data['Cabin'].isnull().sum())
 data['Cabin_dep'] = [cabin_no[0] for cabin_no in data['Cabin'].astype("str")]
data['Cabin_dep'] = data['Cabin_dep'].astype("category" )
import numpy as np
import pandas as pd
import tensorflow as tf<import_modules>
indices_age = data.Age[data['Age'].isnull() == False].index
z_scores_age = np.abs(zscore(data.Age[data['Age'].isnull() == False]))
z_scores = pd.DataFrame(columns=['Age','SibSp','Parch','Fare'])
 z_scores['SibSp'] = zscore(data.SibSp)
 z_scores['Parch'] = zscore(data.Parch)
 z_scores['Fare'] = zscore(data.Fare)
 z_scores['Age'] = zscore(data.Age)
 outlier_rows, outlier_cols = np.where(z_scores > 3)
 outliers = {}
for i,col in enumerate(outlier_rows):
if outlier_rows[i] not in outliers:
outliers[outlier_rows[i]] = [outlier_cols[i]]
else:
outliers[outlier_rows[i]].append(outlier_cols[i])
 to_del = []
for outlier in outliers:
if len(outliers[outlier])>= 2 and outlier >= test_orig.shape[0]:
to_del.append(outlier)
 print(len(to_del))
for i in to_del:
data.drop(data.index[i], inplace=True )
Activation, LeakyReLU, Flatten, Dropout, BatchNormalization 
<load_from_csv>
data.isnull().sum()
train_datas=pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 val_datas = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv" )<split>
data.isnull().sum()
datas = pd.concat([train_datas,val_datas],axis=0)
 print(datas.shape)
 datas_X = np.array(datas.drop("label",axis=1),dtype=np.float32)
 datas_Y = np.array(datas[["label"]],dtype=np.int32)
 train_X,val_X,train_Y,val_Y = train_test_split(datas_X,datas_Y,test_size=0.2,shuffle=True )<train_model>
def create_title(name_col):
for name in name_col:
matches = re.findall("\w+\.",name)
 yield matches[0]
data['Title'] = list(create_title(data['Name']))
data['Title'] = data['Title'].replace({'Mlle.':'Miss.','Ms.':'Miss.','Jonkheer.':"Mr.","Major.":"Mr.","Countess.":"Mrs.","Don.":"Mr.", "Dona.":"Mrs.","Sir.":"Mr.","Mme.":"Miss.", "Col.":"Military", "Major.":"Military","Capt.":"Military", "Lady.":"Mrs."})
 print(data['Title'].value_counts())
 x = np.arange(0,14,2)
 plt.bar(x - 0.25, data.Title[data['Survived'] == 0].value_counts().sort_index() , color = "r", width = 0.5)
 plt.bar(x[:6] + 0.25, data.Title[data['Survived'] == 1].value_counts().sort_index() , color = "b", width = 0.5)
 plt.xticks(x,["Dr.", "Master.", "Military","Miss.","Mr.","Mrs.","Revenant"])
 plt.legend(["Died","Survived"])
 plt.show()
train_X = train_X / 255.0
val_X = val_X / 255.0
train_X = np.reshape(train_X,(-1,28,28,1))
val_X = np.reshape(val_X,(-1,28,28,1))<import_modules>
data = pd.get_dummies(data.drop(['Cabin','Ticket','Name'], axis = 1))
data.head()
from tensorflow.keras.layers import Activation,GlobalAveragePooling2D<choose_model_class>
train, test = [x for _, x in data.groupby(data['Survived'].isnull())]
train = train.drop(['index', 'PassengerId'], axis = 1)
 test = test.drop(['index', 'PassengerId'], axis = 1)

train.Survived.value_counts()
model = Sequential()
model.add(Conv2D(64,(7,7), padding='same',activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(128,(3,3), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Conv2D(128,(3,3), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(2, 2))
model.add(Conv2D(256,(1,1), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(256,(3,3), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(512,(3,3), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Conv2D(512,(1,1), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Conv2D(128,(5,5), padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(256,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64,activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss="sparse_categorical_crossentropy",metrics=["accuracy"])
 print(model.summary() )<choose_model_class>
X_train = train.drop(['Survived'], axis = 1)
 y_train = train['Survived']
X_test = test.drop(['Survived'], axis = 1 )
datagen = ImageDataGenerator(
 rotation_range = 20,
width_shift_range = 0.3,
height_shift_range = 0.3,
shear_range = 0.2,
zoom_range = 0.3,
horizontal_flip = False )<choose_model_class>
kern_ridge = KernelRidge()
params = {'alpha': [0.01,0.1, 1.0],
'coef0': [0,0.1,1],
'degree': [1,2,3],
'gamma': [0.01,1,10],
'kernel': ['rbf']
}
grid_params = {
'estimator':kern_ridge,
'param_grid':params,
'n_jobs':5,
'iid':False,
'verbose':True,
'scoring':'neg_mean_squared_error',
'cv':10
}
y_preds_ridge, ridge_model = run_model(train, test, grid_params, file_name = "submission_ridge.csv", regression = True )
annealer = LearningRateScheduler(lambda x: 0.008 * 0.95 ** x)
 epochs = 10
batch_size = 128<train_model>
log_reg_model = LogisticRegression()
log_reg_params = {'C': [0.1,1.0,10,100],
'class_weight': [None],
'dual': [None],
'fit_intercept': [True,False],
'max_iter': [10000],
'multi_class': ['ovr'],
'class_weight':[class_weights],
'n_jobs': [1],
'penalty': ['l1','l2'],
'random_state': [123],
'tol': [0.0001, 0.0005,0.001],
'solver':['saga'],
'warm_start': [False]}
grid_params['estimator'] = log_reg_model
grid_params['param_grid'] = log_reg_params
grid_params['scoring'] = 'accuracy'
test_preds_log_reg, log_reg_model = run_model(train, test, grid_params, file_name = "submission_log_reg.csv" )
model.fit_generator(datagen.flow(train_X,train_Y, batch_size=batch_size),
epochs = epochs, steps_per_epoch = math.ceil(train_X.shape[0]*1.0/batch_size),
validation_data =(val_X,val_Y), callbacks=[annealer],verbose=1 )<load_from_csv>
xgb_model = XGBClassifier()
params_xgb = {
'base_score':[0.3,0.5],
'colsample_bytree':[0.4,0.7],
'gamma':[0.01,0.5,0.9],
'min_child_weight':[1,3],
'learning_rate':[0.01,0.1,1],
'max_depth':[3,4,5],
'n_estimators':[500],
'reg_alpha':[1e-5, 0.1],
'reg_lambda':[1e-5, 0.1],
'subsample':[0.8]
}
grid_params['estimator'] = xgb_model
grid_params['param_grid'] = params_xgb
grid_params['n_jobs'] = 5
test_preds_xgb, xgb_model = run_model(train, test, grid_params, file_name = "submission_xgb.csv")
test_csv = pd.read_csv(".. /input/Kannada-MNIST/test.csv" )<prepare_x_and_y>
test_gp = test.copy()
train_gp = train.copy()
data_gp = pd.concat([test_gp,train_gp])
 data_gp[['Age','Fare','SibSp','Parch']] = scale(data_gp[['Age','Fare','SibSp','Parch']])
 print(data_gp.head())
 data_gp['Survived'].replace({1:1, 0:-1})
 train_gp, test_gp = [x for _, x in data_gp.groupby(data_gp['Survived'].isnull())]
print(train_gp.shape )
data_test = np.array(test_csv.drop("id",axis=1),dtype=np.float32)
 data_test = np.reshape(data_test,(-1,28,28,1))
X_test = data_test / 255.0
print(X_test.shape,X_test.dtype)
<predict_on_test>
params = {'copy_X_train': [True],
'kernel': [RBF(1.0),RBF(0.01),RBF(0.001),RBF(10), RBF(5), RBF(3), RBF(2)],
'max_iter_predict': [100],
'multi_class': ['one_vs_rest'],
'n_jobs': [3],
'n_restarts_optimizer': [0],
'optimizer': ['fmin_l_bfgs_b'],
'random_state': [123],
'warm_start': [False]
}
grid_params['estimator'] = GaussianProcessClassifier()
grid_params['param_grid'] = params
test_preds_gauss, gp_model = run_model(train_gp, test_gp, grid_params, file_name = "submission_gauss_process.csv" )
results = np.zeros(( X_test.shape[0],10))
results = model.predict(X_test)
 results = np.argmax(results,axis = 1 )<save_to_csv>
params = {'C': [0.01,0.1,0.5,1.0,10,50],
'class_weight': [class_weights],
'coef0': [0.0,0.1,0.5,1],
'decision_function_shape': ['ovr'],
'degree': [2,3],
'gamma': ['auto_deprecated'],
'kernel': ['rbf','poly'],
'max_iter': [-1],
'probability': [False],
'random_state': [123],
'shrinking': [True],
'tol': [0.001,0.003,0.005,0.01]}
grid_params['estimator'] = SVC()
grid_params['param_grid'] = params
test_preds_svm, svm_model = run_model(train_gp, test_gp, grid_params, file_name = "submission_gauss_process.csv" )
submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 submission['label'] = results
submission.to_csv("submission.csv",index=False )<set_options>
rf_model = RandomForestClassifier()
params_rf = {
'bootstrap': [True,False],
'class_weight': [None],
'criterion': ['gini','entropy'],
'max_depth': [None],
'max_features': ['auto'],
'max_leaf_nodes': [None],
'min_impurity_decrease': [0.0,0.01,0.1,1],
'min_impurity_split': [None],
'min_samples_leaf': [1,5,10,15],
'min_samples_split': [2,5,10],
'min_weight_fraction_leaf': [0.0],
'n_estimators': [5,10,20,100],
'n_jobs': [None],
'oob_score': [False],
'random_state': [123]
}
grid_params['estimator'] = rf_model
grid_params['param_grid'] = params_rf
test_preds_rf, rf_model = run_model(train, test, grid_params, file_name = "submission_rf.csv" )
<load_from_csv><EOS> 
svm_model.set_params(probability = True)
 vc_model = VotingClassifier([('rf',rf_model),('xgb',xgb_model),('gp',gp_model),('svm',svm_model)], voting = 'soft')
 X_train = train_gp.drop(['Survived'], axis = 1)
 y_train = train_gp['Survived']
X_test = test_gp.drop(['Survived'],axis = 1)
 vc_model.fit(X_train, y_train)
 preds_train = vc_model.predict(X_train)
 print(confusion_matrix(y_train, preds_train))
y_preds_test = vc_model.predict(X_test)
 create_submission(test_gp,y_preds_test, "submission_vc_soft.csv" )