Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Python version: {}".format(sys.version))
print("pandas version: {}".format(pd.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("NumPy version: {}".format(np.__version__))
print("SciPy version: {}".format(sp.__version__))
print("IPython version: {}".format(IPython.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
print('-'*25)
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
RANDOM_SEED = 19901129
np.random.seed(RANDOM_SEED )
train = pd.read_csv(DATA_PATH + "train.csv")
 test = pd.read_csv(DATA_PATH + "test.csv")
 train["id_code"] = train["id_code"].apply(lambda x: x + ".png")
 test["id_code"] = test["id_code"].apply(lambda x: x + ".png")
 train['diagnosis'] = train['diagnosis'].astype('str')
 train.head(10 )<define_variables>
train = pd.read_csv('.. /input/train.csv', header = 0, dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64})
 full_data = [train, test]
print(train.info() )
batch_size_train = 32
batch_size_test = 1
train_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.1,
horizontal_flip=True,
vertical_flip=True,
rotation_range=20,
zoom_range= 0.2,
featurewise_center=True,
featurewise_std_normalization=True,
zca_whitening=True,
width_shift_range=0.2,
height_shift_range=0.2,)
 train_gen = train_datagen.flow_from_directory(directory=train_dest,
batch_size= batch_size_train,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
subset='training')
 valid_gen = train_datagen.flow_from_directory(directory=train_dest,
batch_size= batch_size_train,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
subset='validation')
 test_datagen = ImageDataGenerator(rescale=1./255)
 test_gen = test_datagen.flow_from_directory(directory=test_dest,
batch_size= batch_size_test,
class_mode= None,
target_size=(IMAGE_SIZE, IMAGE_SIZE),
shuffle=False )<compute_test_metric>
train['Name_length'] = train['Name'].apply(len)
 test['Name_length'] = test['Name'].apply(len)
 train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 train['CategoricalFare'] = pd.qcut(train['Fare'], 4)
 for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 train['CategoricalAge'] = pd.cut(train['Age'], 5)
 def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
dataset['Age*Class'] = dataset['Age'] * dataset['Pclass']
def custom_loss(y_true, y_pred):

 custom_cce = math.add(math.multiply(y_true, -1*math.log(0.001+ y_pred*0.999)) ,
math.multiply(1-y_true, -1*math.log(1-y_pred*0.999)))
 custom_cce = math.reduce_sum(custom_cce, 1)
 weights = cast(1 + math.square(math.abs(math.argmax(y_pred, axis=1)-math.argmax(y_true, axis=1)))/100, float32)
 cce = CategoricalCrossentropy()
return math.multiply(cce(y_pred,y_true), weights)
<choose_model_class>
PassengerId = test['PassengerId']
feature_columns = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone', 'Age*Class']
train = train[feature_columns + ['Survived']]
test = test[feature_columns]
train.head()
print('Creating model...', end=' ')
 input_tensor = Input(shape=(IMAGE_SIZE,IMAGE_SIZE,3))
conv_base = ResNet50(include_top=False, weights=None, input_tensor=input_tensor)
 conv_base.load_weights('.. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
 a_map = layers.Conv2D(516, 1, strides=(1, 1), padding="same", activation='relu' )(conv_base.output)
 a_map = layers.Conv2D(1, 1, strides=(1, 1), padding="same", activation='relu' )(a_map)
 a_map = layers.Conv2D(2048, 1, strides=(1, 1), padding="same", activation='sigmoid' )(a_map)
 res = layers.Multiply()([conv_base.output, a_map])
 x = GlobalAveragePooling2D()(res)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(5, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, predictions)
 model.summary()
print("Done !" )<choose_model_class>
classifiers = []
ntrain = train.shape[0]
ntest = test.shape[0]
kfold = StratifiedKFold(n_splits=10)
 y_train = train['Survived'].ravel()
x_train = train.drop(['Survived'], axis=1 ).values
x_train_columns = train.drop(['Survived'], axis=1 ).columns
x_test = test.values
def classifier_name(clf):
if 'ABCMeta' == clf.__class__:
return clf.__name__
else:
return clf.__class__.__name__
train_steps = train_gen.n//train_gen.batch_size
val_steps = valid_gen.n//valid_gen.batch_size
for layer in model.layers:
layer.trainable = False
for i in range(-9, 0):
model.layers[i].trainable = True
optimizer = Adam(lr=1e-4)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 history_warmup = model.fit_generator(train_gen,
steps_per_epoch=train_steps,
validation_data=valid_gen,
validation_steps=val_steps,
epochs=2 ).history<choose_model_class>
rf_params = {
'n_jobs': -1,
'n_estimators': 500,
'warm_start': False,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
'verbose': 0,
'random_state': RANDOM_SEED
}
rf = RandomForestClassifier(**rf_params)
 classifiers.append(rf )
for layer in model.layers:
layer.trainable = True
earlystopper = EarlyStopping(monitor='val_loss', patience=7, verbose=1, restore_best_weights=True)
 reducel = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.3, min_lr=1e-6)
 optimizer = Adam(1e-4)
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 history = model.fit_generator(train_gen,
steps_per_epoch=train_steps,
validation_data=valid_gen,
validation_steps=val_steps,
epochs=25,
callbacks=[reducel, earlystopper] ).history<find_best_model_class>
svm_params = {'probability': True,
'random_state': RANDOM_SEED}
svm = SVC(**svm_params)
 classifiers.append(svm )
def show_image_mask(path):
im = imread(path)
 exctraction_model = Model(input_tensor, res)
 im = np.array([im])
 w = exctraction_model.predict(im)
 im_res = np.squeeze(im)[:,:,0]
im_w = np.power(resize(np.mean(w[0], axis=2),(224, 224)) ,4)
 imshow(np.multiply(im_w, im_res), cmap = 'gray' )<define_variables>
lr_params = {
'solver': 'liblinear',
'random_state': RANDOM_SEED
}
lr = LogisticRegression(**lr_params)
 classifiers.append(lr )
final_valid_datagen = ImageDataGenerator(rescale=1./255)
 final_valid_gen = train_datagen.flow_from_directory(directory= train_dest,
batch_size= 1,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
shuffle=False)
 STEP_SIZE_TEST = final_valid_gen.n//final_valid_gen.batch_size
print('Start predictions...', end=' ')
 preds = model.predict_generator(final_valid_gen, steps=STEP_SIZE_TEST)
 print('Done !')
 predictions = [np.argmax(pred)for pred in preds]
gt= list(final_valid_gen.labels )<save_to_csv>
kn_params = {'n_neighbors': 3}
kn = KNeighborsClassifier(**kn_params)
 classifiers.append(kn )
test_gen.reset()
STEP_SIZE_TEST = test_gen.n//test_gen.batch_size
print("Predictions begins...")
 preds = model.predict_generator(test_gen, steps=STEP_SIZE_TEST)
 print("Predictions done !")
 predictions = [np.argmax(pred)for pred in preds]
filenames = test_gen.filenames
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])
 results['id_code'] = results['id_code'].apply(lambda x: x.split('/')[1])
 results.to_csv('submission.csv',index=False)
 results.head(10 )<count_values>
gb_params = {'random_state':RANDOM_SEED}
gb = GradientBoostingClassifier(**gb_params)
 classifiers.append(gb )
df = pd.read_csv("submission.csv")
 print(df["diagnosis"].value_counts())
 print(preds )<load_pretrained>
dt_params = {'random_state':RANDOM_SEED}
dt = DecisionTreeClassifier(**dt_params)
 classifiers.append(dt )
if os.path.exists('valid/'):
shutil.rmtree('valid/')
 if os.path.exists(train_dest):
shutil.rmtree(train_dest)
 if os.path.exists(test_dest):
shutil.rmtree(test_dest )<load_from_csv>
ab_params = {'random_state':RANDOM_SEED}
ab = AdaBoostClassifier(**ab_params)
 classifiers.append(ab )
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(test_df.shape )<choose_model_class>
gnb_params = {}
gnb = GaussianNB(**gnb_params)
 classifiers.append(gnb )
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(im_size,im_size,3)
)
 def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
return model
model = build_model()
model.load_weights('.. /input/aptos2019traineddenset/model.h5')
 model.summary()<save_to_csv>
ld_params = {}
ld = LinearDiscriminantAnalysis(**ld_params)
 classifiers.append(ld )
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False)
 print(test_df.head() )<import_modules>
qd_params = {}
qd = QuadraticDiscriminantAnalysis(**qd_params)
 classifiers.append(qd )
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import pandas as pd
import os
import numpy as np
import pandas as pd
import glob
import matplotlib.pyplot as plt
import imagehash
import psutil
from PIL import Image
from joblib import Parallel, delayed
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time 
<feature_engineering>
xgb_params = {'random_state':RANDOM_SEED}
xgb = XGBClassifier(**xgb_params)
 classifiers.append(xgb )
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )<define_variables>
_, NUMBER_OF_FEATURES = x_train.shape
def create_network() :
network = Sequential()
network.add(Dense(units = 9, kernel_initializer = 'uniform', activation = 'relu', input_shape=(NUMBER_OF_FEATURES,)))
 network.add(Dense(units = 9, kernel_initializer = 'uniform', activation = 'relu'))
network.add(Dense(units = 5, kernel_initializer = 'uniform', activation = 'relu'))
network.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
network.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
 return network
NN_classifier = KerasClassifier(build_fn=create_network,
epochs=200,
batch_size=32,
verbose=1)
 classifiers.append(NN_classifier )
bs = 32
sz=245<compute_test_metric>
def evaluate_with_cross_validation(classifiers, test_size, split_num, X, y):
ret_results = {}
cv_results = []
for classifier in tqdm(classifiers):
clf_name = classifier_name(classifier)
 print('cross validate with {0}'.format(clf_name))
n_jobs = None if clf_name in ['KerasClassifier'] else -1
cv_results.append(cross_val_score(classifier, X, y = y, scoring = "accuracy", cv = kfold, n_jobs=n_jobs))
cv_means = []
cv_std = []
for index, cv_result in enumerate(cv_results):
mean_accuracy = cv_result.mean()
std = cv_result.std()
cv_means.append(mean_accuracy)
 cv_std.append(std)
 classifier = classifiers[index]
clf_name = classifier_name(classifier)
 ret_results[clf_name] =(mean_accuracy, classifier)
 print('{0:30s}: {1:.4f}(±{2:.4f})'.format(clf_name, mean_accuracy, std))
cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm": list(ret_results.keys())})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores")
 return ret_results
split_num = 10
test_size = 0.2
classifier_results = evaluate_with_cross_validation(classifiers, test_size, split_num, x_train, y_train )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.argmax(y_hat,1), y, weights='quadratic'),device='cuda:0' )<choose_model_class>
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,2],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsadaDTC.fit(x_train,y_train)
 ada_best = gsadaDTC.best_estimator_
ada_accuracy = gsadaDTC.best_score_
classifier_results['ada_best'] =(ada_accuracy, ada_best)
 ada_accuracy
learn= cnn_learner(data, base_arch=models.vgg19_bn, metrics = [accuracy,quadratic_kappa] )<train_model>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsExtC.fit(x_train,y_train)
 ext_best = gsExtC.best_estimator_
ext_accuracy = gsExtC.best_score_
classifier_results['ext_best'] =(ext_accuracy, ext_best)
 ext_accuracy
learn.fit_one_cycle(4, max_lr=1e-2)
 learn.recorder.plot_losses()<load_from_csv>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsRFC.fit(x_train,y_train)
 RFC_best = gsRFC.best_estimator_
rfc_accuracy = gsRFC.best_score_
classifier_results['RFC_best'] =(rfc_accuracy, RFC_best)
 rfc_accuracy
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection/',folder='test_images',suffix='.png'))
preds,y = learn.get_preds(DatasetType.Test )<save_to_csv>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsGBC.fit(x_train,y_train)
 GBC_best = gsGBC.best_estimator_
gs_accuracy = gsGBC.best_score_
classifier_results['GBC_best'] =(gs_accuracy, GBC_best)
 gs_accuracy
sample_df.diagnosis = preds.argmax(1)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )<set_options>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)
 gsSVMC.fit(x_train,y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC_accuracy = gsSVMC.best_score_
classifier_results['SVMC_best'] =(gsSVMC_accuracy, SVMC_best)
 gsSVMC_accuracy
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
enable_classifiers_num = 5
enable_classifiers = sorted(classifier_results.items() , key=lambda kv: kv[1][0], reverse=True)[:enable_classifiers_num]
print('Top {0:,} classifiers'.format(enable_classifiers_num ).upper())
 for clf_name, tup in enable_classifiers:
print('\t{0:30s}: {1:.5f}'.format(clf_name, tup[0]))
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt<set_options>
estimators = [(n, tup[1])for n, tup in enable_classifiers]
vc_params = {
'estimators': estimators,
'voting': 'soft',
'n_jobs': -1,
'flatten_transform': True
}
voting_classifier = VotingClassifier(**vc_params)
 voting_classifier = voting_classifier.fit(x_train, y_train)
 cv_result = cross_val_score(voting_classifier, x_train, y = y_train, scoring = "accuracy", cv = kfold, n_jobs=-1)
 print('{0:30s}: {1:.4f}(±{2:.4f})'.format(classifier_name(voting_classifier), cv_result.mean() , cv_result.std()))
<feature_engineering><EOS> 
best_classifier = voting_classifier
best_classifier.fit(x_train, y_train)
 predictions = best_classifier.predict(x_test)
 submission_file = pd.DataFrame({ 'PassengerId': PassengerId,
'Survived': predictions })
 ts = datetime.now().strftime("%Y%m%d%H%M")
 filename = 'submission_file_{0}.csv'.format(ts)
 submission_file.to_csv(filename, index=False)
 submission_file