Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
%matplotlib inline
sns.set()
rcParams['figure.figsize'] = 12,8
m.compile(loss="binary_crossentropy",optimizer='adam',metrics=["accuracy"])
 s=time.time()
h=m.fit(X,Y,batch_size=128,validation_split=0.2,epochs=100)
 e=time.time()
t=e-s
print("Addestramento completato in %d minuti e %d secondi" %(t/60,t*60))<find_best_params>
train_df = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
 train_df.head()
acc=h.history['acc']
val_acc=h.history['val_acc']
loss=h.history['loss']
val_loss=h.history['val_loss']<save_to_csv>
print('Train dataset has only {} unique tickets'.format(len(train_df['Ticket'].unique())))
 print('-'*40)
 print('Test dataset has only {} unique tickets'.format(len(test_df['Ticket'].unique())) )
pred=m.predict(X_test)
 ids=[]
label=[]
a=0
for i in tqdm(os.listdir(test_path)) :
id=i
ids.append(id)
 label.append(pred[a])
 a=a+1
label=np.array(label,dtype='float64')
 out=pd.DataFrame({'id': ids,'has_cactus':label[:,0]})
 out.to_csv('cactus_identifier_net.csv',index=False,header=True )<set_options>
new_train_df = train_df.copy()
new_test_df = test_df.copy()
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
def total_famil_members(df):
return df['SibSp']+df['Parch']+1
from fastai.vision import *
from fastai import *
from fastai.metrics import error_rate
import pandas as pd
import torch<load_from_csv>
for data in [new_train_df, new_test_df]:
data['Members'] = total_famil_members(data)
 data['Adjusted_Fare'] = data['Fare']/data['Members']
data['Title'] = data['Name'].apply(get_title )
path =".. /input/"
train_df=pd.read_csv(path+"train.csv")
 test_df=pd.read_csv(path+"sample_submission.csv")
<load_from_csv>
for data in [new_train_df, new_test_df]:
data['Title'] = data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs')
bs = 128
data = ImageDataBunch.from_csv(path=path, folder='train/train', csv_labels='train.csv', ds_tfms=get_transforms() , size=32, bs=bs ).normalize(imagenet_stats)
<define_variables>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for data in [new_train_df, new_test_df]:
data['Title'] = data['Title'].map(title_mapping)
 data['Title'] = data['Title'].fillna(0)
 data['Embarked'] = data['Embarked'].fillna('S')
 data['Sex'] = data['Sex'].map({'female': 0, 'male': 1} ).astype(int)
data.show_batch(rows=3, figsize=(7,6))<find_best_params>
for data in [new_train_df, new_test_df]:
data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
data.classes, data.c, len(data.train_ds), len(data.valid_ds )<choose_model_class>
data_train = new_train_df[~new_train_df['Age'].isnull() ]
data_test = new_test_df[~new_test_df['Age'].isnull() ]
learn = cnn_learner(data, models.resnet50, metrics=error_rate, model_dir="/tmp/model/" )<train_model>
X_train = data_train[['Pclass','Sex','SibSp','Parch','Title']]
y_train = data_train['Age']
X_test = data_test[['Pclass','Sex','SibSp','Parch','Title']]
y_test = data_test['Age']
X_train.head()
learn.fit_one_cycle(6 )<train_model>
model_age_prediction = RandomForestRegressor(n_estimators=900, max_depth=6, min_samples_leaf=0.001, random_state=100)
 model_age_prediction.fit(X_train, y_train )
learn.fit_one_cycle(5, max_lr=slice(3e-5,3e-4))<choose_model_class>
y_predict = model_age_prediction.predict(X_test )
interp = ClassificationInterpretation.from_learner(learn )<predict_on_test>
print('test score is: {}'.format(r2_score(y_test, y_predict)))
 print('training score is: {}'.format(r2_score(y_train, model_age_prediction.predict(X_train))))
a,b,c=learn.predict(open_image(".. /input/test/test/000940378805c44108d287872b2f04ce.jpg"))
print(c)
 print(c[1].numpy())
<predict_on_test>
train_missing_predicted = model_age_prediction.predict(new_train_df[new_train_df['Age'].isnull() ][['Pclass','Sex','SibSp','Parch','Title']])
 test_missing_predicted = model_age_prediction.predict(new_test_df[new_test_df['Age'].isnull() ][['Pclass','Sex','SibSp','Parch','Title']] )
def pred(name):
a,b,c=learn.predict(open_image(".. /input/test/test/"+name))
return c[1].numpy()<feature_engineering>
new_train_df['Age'][np.isnan(new_train_df['Age'])] = train_missing_predicted
new_test_df['Age'][np.isnan(new_test_df['Age'])] = test_missing_predicted
test_df["has_cactus"]=test_df["id"].apply(lambda x:pred(x))<save_to_csv>
new_test_df[new_test_df['Adjusted_Fare'].isna() ]
test_df.to_csv('submission.csv',index=False )<set_options>
new_test_df['Fare'] = new_test_df['Adjusted_Fare']*new_test_df['Members']
new_test_df.info()
%matplotlib inline
print(os.listdir(".. /input")) 
<train_model>
data_model = new_train_df[['Survived', 'Pclass', 'Sex', 'Age', 'Embarked', 'Members', 'Adjusted_Fare', 'Title']]
data_model.head()
def read_pix(jpg_dir):
filenames = glob.glob(os.path.join(jpg_dir, '*.jpg'))
img_array = np.zeros(( len(filenames), 32, 32, 3), dtype=int)
 img_index = []
for idx, filename in enumerate(filenames):
im_tmp = matplotlib.image.imread(filename)
 img_array[idx, :, :, :] = np.array(im_tmp, dtype=int)
 img_index.append(os.path.basename(filename))
return img_array, img_index
def prepare_data(img_array, img_index, train_response, test_size=0.25):
y_train_series, y_test_series = train_test_split(train_response,
shuffle=True,
random_state=12,
test_size=test_size)
 y_train = y_train_series.values[:, np.newaxis]
y_test = y_test_series.values[:, np.newaxis]
train_index = [img_index.index(idx)for idx in y_train_series.index]
test_index = [img_index.index(idx)for idx in y_test_series.index]
x_train = img_array[train_index, :, :, :]
x_test = img_array[test_index, :, :, :]
return x_train, y_train, x_test, y_test, y_train_series, y_test_series
def simple_oversample(x, y, oversample_class, oversample_factor=3):
new_x = x.copy()
new_y = y.copy()
oversample_mask = np.ravel(y == oversample_class)
 oversample_x = x[oversample_mask, :, :, :]
oversample_y = y[oversample_mask, :]
for i in range(oversample_factor):
new_x = np.concatenate([new_x, oversample_x], axis=0)
 new_y = np.concatenate([new_y, oversample_y], axis=0)
 return new_x, new_y<load_from_csv>
train_X, test_X, train_y, test_y = train_test_split(data_model.drop(['Survived'], axis=1), data_model['Survived'],\
stratify=data_model['Survived'], random_state=123, test_size=0.25 )
train_labels = pd.read_csv('.. /input/train.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train_img_array, train_img_index = read_pix('.. /input/train/train')
 test_img_array, test_img_index = read_pix('.. /input/test/test')
 train_response = train_labels['has_cactus']
train_response.index = train_labels['id']
train_response = train_response.loc[train_img_index]<split>
lr = LogisticRegression()
params_lr = {'C':[0.001, 0.01, 0.1, 1, 10, 100]}
batch_size = 128
num_classes = 2
datagen = ImageDataGenerator(
 rotation_range=90,
width_shift_range=.1,
height_shift_range=.1,
shear_range=.2,
zoom_range=.1,
horizontal_flip=True,
vertical_flip=True,
fill_mode='nearest', 
)
 x_train, y_train, x_test, y_test, y_train_series, y_test_series = prepare_data(train_img_array, train_img_index, train_response, test_size=0.2)
 x_train, y_train = simple_oversample(x_train, y_train, oversample_class=0, oversample_factor=2)
 x_test, y_test = simple_oversample(x_test, y_test, oversample_class=0, oversample_factor=2)
 print("Number of 'no cactus' samples in y_train: {}".format(( y_train == 0 ).sum()))
print("Number of 'cactus' samples in y_train: {}".format(( y_train == 1 ).sum()))<categorify>
lr_cv = GridSearchCV(lr, params_lr, n_jobs=-1, cv=5 )
x_train = x_train / 255
x_test = x_test / 255
train_generator = datagen.flow(x_train, y_train)
 test_datagen = ImageDataGenerator()
validation_generator = test_datagen.flow(x_test, y_test )<choose_model_class>
lr_cv.fit(train_X, train_y )
def convnet_model() :
model = Sequential()
model.add(Conv2D(32,(5, 5), input_shape=(32, 32, 3), activation='relu'))
model.add(Conv2D(32,(5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.5))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-4, decay=1e-6), metrics=['accuracy'])
 return model<train_model>
lr_cv_best_model = pd.DataFrame({'Params':lr_cv.best_params_.values() , 'best_score':lr_cv.best_score_,\
'Train Score':accuracy_score(train_y, lr_cv.best_estimator_.predict(train_X)) ,\
'Test Score': accuracy_score(test_y, lr_cv.best_estimator_.predict(test_X)) } )
model = convnet_model()
hist = model.fit_generator(train_generator,
steps_per_epoch=np.ceil(x_train.shape[0] / 32),
epochs=120,
validation_data=validation_generator,
validation_steps=np.ceil(x_test.shape[0] / 32)
)
 scores = model.evaluate(x_test, y_test, verbose=0)
 print('CNN error: {:.2f}'.format(100-scores[1]*100))<save_to_csv>
svc = LinearSVC()
params_svc = params_lr
probability = model.predict_proba(test_img_array/255)
 res = pd.DataFrame({
'id': test_img_index,
'has_cactus': probability.ravel() ,
})
 res.to_csv('submission.csv', index=False )<import_modules>
svc_cv = GridSearchCV(svc, params_svc, n_jobs=-1 )
import numpy as np
import pandas as pd
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch 
<load_from_csv>
svc_cv.fit(train_X, train_y )
data_folder = Path(".. /input")
 train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<define_variables>
svc_cv_best_model = pd.DataFrame({'Params':svc_cv.best_params_.values() , 'best_score':svc_cv.best_score_,\
'Train Score':accuracy_score(train_y, svc_cv.best_estimator_.predict(train_X)) ,\
'Test Score': accuracy_score(test_y, svc_cv.best_estimator_.predict(test_X)) } )
test_img = ImageList.from_df(test_df, path=data_folder/'test', folder='test' )<categorify>
max_depth =[]
min_samples_leaf = []
cv_rf_scores = []
test_roc_scores = []
train_roc_scores = []
test_acc_scores = []
train_acc_scores = []
for depth in [5,6,7,8,9]:
for samples_leaf in [0.009, 0.01]:
rf = RandomForestClassifier(n_estimators=400, n_jobs=-1, max_features='log2', max_depth=depth, min_samples_leaf=samples_leaf, random_state=100)
 rf.fit(train_X, train_y)
 cv_rf_scores.append(cross_val_score(rf, train_X, train_y, cv=10 ).mean())
 max_depth.append(depth)
 min_samples_leaf.append(samples_leaf)
 test_roc_scores.append(roc_auc_score(test_y, rf.predict(test_X)))
 test_acc_scores.append(accuracy_score(test_y, rf.predict(test_X)))
 train_roc_scores.append(roc_auc_score(train_y, rf.predict(train_X)))
 train_acc_scores.append(accuracy_score(train_y, rf.predict(train_X)) )
src =(ImageList.from_df(train_df, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_img)
 )<load_pretrained>
rf_cv_scores = pd.DataFrame({'Max_depth':max_depth, 'Min_samples_leaf':min_samples_leaf, 'CV_Scores':cv_rf_scores,\
'Test_roc_score':test_roc_scores, 'Train_roc_scores':train_roc_scores,\
'Test_acc_score':test_acc_scores, 'Train_acc_scores':train_acc_scores} )
train_img=src.databunch('.',bs=50 )<define_variables>
rf_cv_scores_sorted = rf_cv_scores.sort_values(by='CV_Scores' ).reset_index()
rf_cv_scores_sorted.head(18 )
train_img.show_batch()<feature_engineering>
classifiers = [('Logistic Regression',LogisticRegression(C=0.1)) ,\ 
('SVC', LinearSVC(C=0.01)) ,\ 
('Random Forest', RandomForestClassifier(n_estimators=400, n_jobs=-1, max_features='log2', max_depth=7, min_samples_leaf=0.01, random_state=100)) ]
tfms=get_transforms(flip_vert=True )<normalization>
vc = VotingClassifier(estimators=classifiers, n_jobs=-1 )
train_img =(src.transform(tfms,size=128)
.databunch('.',bs=50)
 )<choose_model_class>
vc.fit(train_X, train_y )
denselearner = cnn_learner(train_img, models.densenet161, metrics=[FBeta() ,error_rate, accuracy] )<find_best_params>
vc_scores = pd.DataFrame({'Best_score':cross_val_score(vc, train_X, train_y, cv=5 ).mean() ,\
'Train Score':accuracy_score(train_y, vc.predict(train_X)) ,\
'Test Score': accuracy_score(test_y, vc.predict(test_X)) }, index=[0] )
denselearner.lr_find()
denselearner.recorder.plot(suggestion=True )<train_model>
print('ROC_AUC score for VC is {}'.format(roc_auc_score(test_y, vc.predict(test_X))))
print('Accuracy score for VC is {}'.format(accuracy_score(test_y, vc.predict(test_X))))
lr = 7.5e-03
denselearner.fit_one_cycle(5, slice(lr))<find_best_params>
rf1 = RandomForestClassifier(n_estimators=400, n_jobs=-1, max_features='log2', max_depth=7, min_samples_leaf=0.01, random_state=100)
 rf1.fit(train_X, train_y )
denselearner.unfreeze()
denselearner.lr_find()
denselearner.recorder.plot(suggestion=True )<train_model>
print('ROC_AUC score for RF is {}'.format(roc_auc_score(test_y, rf1.predict(test_X))))
print('Accuracy score for RF is {}'.format(accuracy_score(test_y, rf1.predict(test_X))))
denselearner.fit_one_cycle(1, slice(1e-06))<choose_model_class>
results = rf1.predict(new_test_df[['Pclass', 'Sex', 'Age', 'Embarked', 'Members', 'Adjusted_Fare', 'Title']] )
reslearner = cnn_learner(train_img, models.resnet101, metrics=[FBeta() ,error_rate, accuracy] )<find_best_params>
submission = pd.DataFrame({'PassengerId':new_test_df.PassengerId, 'Survived':results} )
reslearner.lr_find()<define_search_space>
submission.to_csv('submission.csv', index=False )
<train_model><EOS> 
submission.to_csv('submission.csv', index=False )