Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
sns.set_style("whitegrid")
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 200)
 warnings.filterwarnings('ignore')
 print(device_lib.list_local_devices())
 config = tf.ConfigProto(device_count={"CPU": 1, "GPU" : 1})
 session = tf.Session(config=config)
 K.set_session(session )
model.load_weights('.. /input/weights/model.hdf5' )<train_model>
test = pd.read_csv(".. /input/test.csv", ",")
 train = pd.read_csv(".. /input/train.csv", ",")
 test["is_test"] = True
train["is_test"] = False
common = pd.concat([test, train],axis=0 ).loc[:,["PassengerId", "Survived", "is_test",
"Age", "Cabin", "Embarked",
"Fare", "Name", "Parch", "Pclass",
"Sex", "SibSp", "Ticket"]]
model.fit(X_train[:128], y_train[:128], epochs=30 )<train_model>
common["Ticket"].count() - len(common["Ticket"].unique() )
checkpoint = ModelCheckpoint(filepath='model.hdf5')
 model.fit(X_train, y_train, epochs=100, batch_size=128, callbacks=[checkpoint] )<load_from_csv>
t = train.groupby(by="Ticket", as_index=False ).agg({"PassengerId" : 'count', "Sex" : lambda x : x[x=="female"].count() })
 t.columns = ["Ticket", "SameTicket", "FemalesOnTicket"]
common = pd.merge(common, t, how="left", on="Ticket")
 common["TicketDigits"] = pd.to_numeric(common["Ticket"].str.split(" " ).str[-1], errors="coerce" ).astype(np.str ).str.len()
common["TicketIsNumber"] = ~common["Ticket"].str.contains("[A-Za-z]", regex = True)
 common["FemalesPerTicketPart"] = common["FemalesOnTicket"]/common["SameTicket"]
sample = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv' )<predict_on_test>
common["DoubleName"] = common["Name"].str.contains("\(")
 common["NameLen"] = common["Name"].str.len()
sample.has_cactus = model.predict(X_test )<save_to_csv>
common["Title"] = common["Name"].str.split(", " ).str[1].str.split(" " ).str[0]
common.loc[common["Title"].str[-1]!=".", "Title"]="Bad"
rare_title = common["Title"].value_counts() [common["Title"].value_counts() < 5].index
common["Title"] = common["Title"].apply(lambda x: 'Rare' if x in rare_title else x)
 titletarget = common.groupby(by="Title", as_index=False ).agg({"Survived" : 'mean'})
 titletarget.columns = ["Title", "TargetByTitle"]
common = pd.merge(common, titletarget, how="left", on="Title" )
sample.to_csv('sample_submission.csv', index=False )<load_from_csv>
common["Family"] = common["Parch"] + common["SibSp"] + 1
common["Alone"] = common["Family"] == 1
input_path = '.. /input/'
train_path = input_path + 'train/train/'
test_path = input_path + 'test/test/'
train_dir=".. /input/train/train"
test_dir=".. /input/test/test"
train_df=pd.read_csv('.. /input/train.csv')
 test_df=pd.read_csv('.. /input/sample_submission.csv')
 train_id = train_df['id']
labels = train_df['has_cactus']
test_id = test_df['id']<split>
cap = train.groupby(by="Cabin", as_index=False ).agg({"PassengerId" : 'count'})
 cap.columns = ["Cabin", "SameCabin"]
common = pd.merge(common, cap, how="left", on="Cabin")
 common["CabinNumber"] = pd.to_numeric(common["Cabin"].str[1:], errors = "coerce")
 common["CabinEven"] = common["CabinNumber"] %2
common["CabinsPerMan"] = common["Cabin"].str.split(" " ).str.len()
common["Deck"] = common["Cabin"].str[0].rank().fillna(-1)
 decktarget = common.groupby(by="Deck", as_index=False ).agg({"Survived" : 'mean'})
 decktarget.columns = ["Deck", "TargetByDeck"]
common = pd.merge(common, decktarget, how="left", on="Deck" )
x_train, x_val, y_train, y_val = train_test_split(train_id, labels, test_size=0.2 )<data_type_conversions>
common['AgeGroup'] = pd.qcut(common['Age'].fillna(common['Age'].mean() ).astype(int), 6)
 agetarget = common.groupby(by="AgeGroup", as_index=False ).agg({"Survived" : 'mean'})
 agetarget.columns = ["AgeGroup", "TargetByAgeGroup"]
common = pd.merge(common, agetarget, how="left", on="AgeGroup")
 common["IsTinyChild"] = common["Age"]<1
common["IsChild"] = common["Age"]<10
common["AverageAge"] = common["Age"] / common["Family"]
def get_images(ids, filepath):
arr = []
for img_id in ids:
img = plt.imread(filepath + img_id)
 arr.append(img)
 arr = np.array(arr ).astype('float32')
 arr = arr / 255
return arr<categorify>
common['FareGroup'] = pd.qcut(common['Fare'].fillna(common['Fare'].mean() ).astype(int), 6)
 faretarget = common.groupby(by="FareGroup", as_index=False ).agg({"Survived" : 'mean'})
 faretarget.columns = ["FareGroup", "TargetByFareGroup"]
common = pd.merge(common, faretarget, how="left", on="FareGroup")
 common["AverageFareByFamily"] = common["Fare"] / common["Family"]
common["AverageFareByTicket"] = common["Fare"] / common["SameTicket"]
common["FareLog"] = np.log(common["Fare"] )
x_train = get_images(ids=x_train, filepath=train_path)
 x_val = get_images(ids=x_val, filepath=train_path)
 test = get_images(ids=test_id, filepath=test_path)
 img_dim = x_train.shape[1:]<define_variables>
pclasstarget = common.groupby(by="Pclass", as_index=False ).agg({"Survived" : 'mean'})
 pclasstarget.columns = ["Pclass", "TargetByPclass"]
common = pd.merge(common, pclasstarget, how="left", on="Pclass")
 Embarkedtarget = common.groupby(by="Embarked", as_index=False ).agg({"Survived" : 'mean'})
 Embarkedtarget.columns = ["Embarked", "TargetByEmbarked"]
common = pd.merge(common, Embarkedtarget, how="left", on="Embarked")
 Sextarget = common.groupby(by="Sex", as_index=False ).agg({"Survived" : 'mean'})
 Sextarget.columns = ["Sex", "TargetBySex"]
common = pd.merge(common, Sextarget, how="left", on="Sex" )
batch_size = 64
epochs = 30
steps = x_train.shape[0] // batch_size<choose_model_class>
allfeatures = [
"PassengerId",
"is_test",
"Survived",
"Age",
"Fare",
"Parch",
"Pclass",
"SibSp",
"Sex",
"Embarked",
"SameTicket",
"FemalesOnTicket",
"SameCabin",
"Deck",
"TargetByDeck",
"TargetByTitle",
"TargetByAgeGroup",
"TargetByFareGroup",
"TargetByPclass",
"TargetByEmbarked",
"TargetBySex",
"Title",
"CabinNumber",
"CabinEven",
"CabinsPerMan",
"DoubleName",
"NameLen",
"TicketDigits",
"TicketIsNumber",
"IsTinyChild",
"IsChild",
"Alone",
"Family",
"AverageAge",
"AverageFareByFamily",
"AverageFareByTicket",
"FemalesPerTicketPart"
]
c = common.loc[:, allfeatures]
c = pd.get_dummies(c, columns=[
"Title",
"Embarked",
"Pclass",
"Sex"
] )
inputs = Input(shape=img_dim)
 densenet121 = DenseNet121(weights='imagenet', include_top=False )(inputs)
 flat1 = Flatten()(densenet121)
 dense1 = Dense(units=256, use_bias=True )(flat1)
 batchnorm1 = BatchNormalization()(dense1)
 act1 = Activation(activation='relu' )(batchnorm1)
 drop1 = Dropout(rate=0.5 )(act1)
 out = Dense(units=1, activation='sigmoid' )(drop1)
 model = Model(inputs=inputs, outputs=out)
 model.compile(optimizer='adam', loss='binary_crossentropy' )<train_model>
c.iloc[:,3:] = Imputer(strategy="most_frequent" ).fit_transform(c.iloc[:,3:])
 dep = c[c["is_test"] == False].loc[:, ["Survived"]]
indep = c[c["is_test"] == False].iloc[:, 3:]
res = c[c["is_test"] == True].iloc[:, 3:]
res_index = c[c["is_test"] == True].loc[:, "PassengerId"]
indep.iloc[:,3:] = StandardScaler().fit_transform(indep.iloc[:,3:])
 res.iloc[:,3:] = StandardScaler().fit_transform(res.iloc[:,3:])
 indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=47 )
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=3, verbose=2, mode='max')
 img_aug = ImageDataGenerator(rotation_range=20, vertical_flip=True, horizontal_flip=True)
 img_aug.fit(x_train)
 model.fit_generator(img_aug.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=steps, epochs=epochs,
validation_data=(x_val, y_val), callbacks=[reduce_lr],
verbose=2 )<predict_on_test>
with tf.device('/device:CPU:0'):
gs1 = Sequential()
gs1.add(Dense(45 ,activation='linear', input_dim=45))
gs1.add(BatchNormalization())
 gs1.add(Dense(9,activation='linear'))
gs1.add(BatchNormalization())
 gs1.add(Dropout(0.4))
gs1.add(Dense(5,activation='linear'))
gs1.add(BatchNormalization())
 gs1.add(Dropout(0.2))
gs1.add(Dense(1,activation='relu',))
gs1.compile(optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0), loss='binary_crossentropy', metrics=['accuracy'])
 gs1.fit(indep_train, dep_train, epochs=500, batch_size=30, validation_data=(indep_test,dep_test), verbose=False)
 g=gs1.predict_classes(res)[:,0]
print(accuracy_score(dep_test, gs1.predict_classes(indep_test)) , accuracy_score(dep_train, gs1.predict_classes(indep_train)) )
test_pred = model.predict(test, verbose=2 )<save_to_csv>
cvscores = []
data = pd.DataFrame()
i=1
with tf.device('/device:CPU:0'):
for train, test in StratifiedKFold(n_splits=5, shuffle=True, random_state=1 ).split(indep, dep.iloc[:,0]):
X = indep.reindex().iloc[train,:]
Y = dep.reindex().iloc[train,0]
Xv = indep.reindex().iloc[test,:]
Yv = dep.reindex().iloc[test,0]
gs1 = Sequential()
gs1.add(Dense(45 ,activation='linear', input_dim=45))
gs1.add(BatchNormalization())
 gs1.add(Dense(9,activation='linear'))
gs1.add(BatchNormalization())
 gs1.add(Dropout(0.4))
gs1.add(Dense(5,activation='linear'))
gs1.add(BatchNormalization())
 gs1.add(Dropout(0.2))
gs1.add(Dense(1,activation='relu',))
gs1.compile(optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0), loss='binary_crossentropy', metrics=['accuracy'])
 gs1.fit(X, Y, epochs=500, batch_size=30, validation_data=(Xv, Yv), verbose=False)
 data[i] = gs1.predict_classes(res)[:,0]
scores = gs1.evaluate(Xv, Yv, verbose=0)
 print(gs1.metrics_names[1], scores[1])
 cvscores.append(scores[1])
 i+=1
mlp_mean = np.mean(cvscores)
 mlp_stdev = np.std(cvscores)
 print(mlp_mean, mlp_stdev)
 g = np.round(data.mean(axis=1))
test_df['has_cactus'] = test_pred
test_df.to_csv('submission.csv', index=False )<import_modules>
;
import os
import numpy as np
from fastai import *
from fastai.vision import *<define_variables>
indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=47)
 gs1 = lgb.LGBMClassifier(max_depth = 7,
lambda_l1 = 0.1,
lambda_l2 = 0.01,
learning_rate = 0.01,
n_estimators = 500, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,
n_jobs = 5)
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], eval_metric='accuracy', verbose=False, early_stopping_rounds=50);
g = gs1.predict(res)
 a = accuracy_score(dep_test, gs1.predict(indep_test))
b = accuracy_score(dep_train, gs1.predict(indep_train))
print(a, b )
train_path = Path('.. /input/train')
 test_path = Path('.. /input/test')
 print(train_path, test_path)
<load_from_csv>
model = []
cvscores = []
for i in range(0,90):
indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)
 gs1 = lgb.LGBMClassifier(max_depth = 7,
lambda_l1 = 0.1,
lambda_l2 = 0.01,
learning_rate = 0.01, num_iterations=20000,
n_estimators = 5000, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,
n_jobs = 5, boosting='dart')
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], eval_metric='accuracy', verbose=False, early_stopping_rounds=50);
model.append(gs1)
 cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))
 data = pd.DataFrame()
te = pd.DataFrame()
for i in range(0,90):
data[i] = model[i].predict(res)
 te[i] = model[i].predict(indep_test)
 g = np.round(data.mean(axis=1))
t = np.round(te.mean(axis=1))
lgb_mean = np.mean(cvscores)
 lgb_stdev = np.std(cvscores)
 print(lgb_mean, lgb_stdev )
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head(10 )<load_from_csv>
result = pd.DataFrame(res_index.astype(np.int), columns=["PassengerId"])
 result["Survived"] = g.astype(np.int)
 result.to_csv("lgbm.csv", ",", index=None )
test_df = pd.read_csv('.. /input/sample_submission.csv')
<categorify>
gs1 = cb.CatBoostClassifier(depth = 9, reg_lambda=0.1,
learning_rate = 0.09,
iterations = 500)
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], verbose=False, early_stopping_rounds=50);
g = gs1.predict(res)
 a = accuracy_score(dep_test, gs1.predict(indep_test))
b = accuracy_score(dep_train, gs1.predict(indep_train))
print(a, b )
test_data = ImageList.from_df(test_df, path = test_path, folder = 'test')
 train_data =(ImageList.from_df(train_df, path = train_path, folder = 'train')
.split_by_rand_pct(0.1)
.label_from_df() 
.add_test(test_data)
.transform(get_transforms() , size = 32)
.databunch(path = '.', bs = 64)
.normalize(imagenet_stats))<choose_model_class>
model = []
cvscores = []
for i in range(0, 90):
indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)
 gs1 = cb.CatBoostClassifier(depth = 9, reg_lambda=0.1,
learning_rate = 0.09,
iterations = 500)
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], verbose=False, early_stopping_rounds=50);
model.append(gs1)
 cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))
 data = pd.DataFrame()
te = pd.DataFrame()
for i in range(0, 90):
data[i] = model[i].predict(res)
 te[i] = model[i].predict(indep_test)
 g = np.round(data.mean(axis=1))
t = np.round(te.mean(axis=1))
cb_mean = np.mean(cvscores)
 cb_stdev = np.std(cvscores)
 print(cb_mean, cb_stdev)
learner = cnn_learner(train_data, models.densenet161, metrics = [error_rate, accuracy] )<train_model>
gs1 = xgb.XGBClassifier(max_depth = 9,
learning_rate = 0.01,
n_estimators = 500, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,
n_jobs = 5)
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], verbose=False, early_stopping_rounds=50);
g = gs1.predict(res)
 a = accuracy_score(dep_test, gs1.predict(indep_test))
b = accuracy_score(dep_train, gs1.predict(indep_train))
print(a, b )
learner.fit_one_cycle(8 )<train_model>
model = []
cvscores = []
for i in range(0, 90):
indep_train, indep_test, dep_train, dep_test = train_test_split(indep, dep, test_size=0.40, random_state=i)
 gs1 = xgb.XGBClassifier(max_depth = 7, reg_lambda = 0.02,
learning_rate = 0.01,
n_estimators = 5000, reg_alpha = 1.1, colsample_bytree = 0.9, subsample = 0.9,
n_jobs = 5)
 gs1.fit(indep_train, dep_train, eval_set=[(indep_test, dep_test)], verbose=False, early_stopping_rounds=50);
model.append(gs1)
 cvscores.append(accuracy_score(dep_test, gs1.predict(indep_test)))
 data = pd.DataFrame()
te = pd.DataFrame()
for i in range(0, 90):
data[i] = model[i].predict(res)
 te[i] = model[i].predict(indep_test)
 g = np.round(data.mean(axis=1))
t = np.round(te.mean(axis=1))
xgb_mean = np.mean(cvscores)
 xgb_stdev = np.std(cvscores)
 print(xgb_mean, xgb_stdev )
<predict_on_test><EOS> 
d = {'Model':["Keras MLP", "LightGBM", "CatBoost", "XGBoost"],
'Mean accuracy': [mlp_mean, lgb_mean, cb_mean, xgb_mean],
'Std.Dev.': [mlp_stdev, lgb_stdev, cb_stdev, xgb_stdev],
'Leaderboard': [0.77033, 0.82296, 0.78947, 0.77511]}
pd.DataFrame(data=d, columns=["Model", "Mean accuracy", "Std.Dev.", "Leaderboard"] ).sort_values("Mean accuracy", ascending=False ).head(10 )