Kaggle competition: Titanic - Machine Learning from Disaster
cat_dog_cols = list(df_external_3_dog.columns)+list(set(df_external_3_cat.columns)- set(df_external_3_dog.columns))
df_external_3_all = pd.concat(( df_external_3_dog,df_external_3_cat),axis=0)[cat_dog_cols]<feature_engineering>
%matplotlib inline
warnings.filterwarnings('ignore' )
breed_id = {}
for id_,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[name] = id_<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.describe()
df_external_3_all['BreedName']= df_external_3_all.BreedName.map(breed_id )<rename_columns>
print(train.isnull().sum() )
df_external_3_all = df_external_3_all.rename(columns={"BreedName":'pred_breed'} )<merge>
train = train.drop(["Ticket"],axis = 1)
 test = test.drop(['Ticket'], axis = 1)
 train = train.drop(["Cabin"],axis = 1)
 test = test.drop(["Cabin"],axis = 1)
 train = train.drop(["CabinBool"],axis = 1)
 test = test.drop(["CabinBool"],axis = 1 )
df_all = df_all.merge(df_external_3_all,how='left',on='pred_breed')
<categorify>
train = train.fillna({"Embarked": "S"} )
df_all_temp = df_all[['Temperment']].copy()
df_all_temp['Temperment'] = df_all_temp['Temperment'].apply(lambda x: [e.strip(',')for e in str(x ).split() ])
 df_all_temp = df_all_temp.drop('Temperment', 1 ).join(df_all_temp.Temperment.str.join('|' ).str.get_dummies())
 df_all_temp = df_all_temp.add_prefix('Temperment_')
 df_all = pd.concat(( df_all,df_all_temp),axis=1)
 del df_all['Temperment']<merge>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace(['Countess', 'Lady', 'Sir'], 'Royal')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
rescuer_count = df_all.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
df_all = df_all.merge(rescuer_count, how='left', on='RescuerID' )<define_variables>
title_map = {"Master":1,"Miss":2,"Mr":3,"Mrs":4,"Rare":5,"Royal":5}
for dataset in combine:
dataset["Title"] = dataset["Title"].map(title_map)
 dataset["Title"] = dataset["Title"].fillna(0)
 train.head()
groupby_columns = [['is_dog','Breed1'],
['State'],
['is_dog','Gender','Age_bins'],
['is_dog','MaturitySize'],
['is_dog','FurLength'],
['is_dog','Breed1','Vaccinated', 'Dewormed', 'Sterilized', 'Health',]
]
aggregate_columns = ['Age',
'Name_len',
'Color1',
'Color2',
'Color3',
'MaturitySize',
'FurLength',
'Vaccinated',
'Dewormed',
'Sterilized',
'Health',
'Quantity',
'Fee',
'VideoAmt',
'PhotoAmt',
'state_gdp',
'state_population',
'len_words_Description',
'len_isupper_Description_to_characters',
'len_islower_Description_to_characters',
'len_punctuation_Description_to_characters',
'len_letters_Description_to_characters',
'len_characters_Description_to_words',
'RescuerID_COUNT',
]<merge>
Master_age = train[train["Title"]==1]["AgeGroups"].mode()
Ms_age = train[train["Title"]==2]["AgeGroups"].mode()
Mr_age = train[train["Title"]==3]["AgeGroups"].mode()
Mrs_age = train[train["Title"]==4]["AgeGroups"].mode()
Rare_age = train[train["Title"]==5]["AgeGroups"].mode()
Royal_age = train[train["Title"]==6]["AgeGroups"].mode()
print(Master_age)
 print(Ms_age)
 print(Mrs_age)
 train.head()
for _groupby in groupby_columns:
_aggregate = [e for e in aggregate_columns if e not in _groupby]
_aggregate = {e:['mean','std','min','max'] for e in _aggregate}
gr = df_all.groupby(_groupby ).agg(_aggregate)
 gr.columns = ['{}_{}_{}'.format("_".join(_groupby),e[0],e[1])for e in gr.columns]
all_columns.extend(gr.columns)
 gr = gr.reset_index()
df_all = pd.merge(df_all,gr,on=_groupby,how='left' )<define_variables>
title_age_map = {1: "Small children", 2: "Teens", 3: "Young Adults", 4: "Adults", 5: "Adults", 6: "Adults"}
for x in range(len(train["AgeGroups"])) :
if train["AgeGroups"][x] == "Unknown":
train["AgeGroups"][x] = title_age_map[train["Title"][x]]
for x in range(len(test["AgeGroups"])) :
if test["AgeGroups"][x] == "Unknown":
test["AgeGroups"][x] = title_age_map[train["Title"][x]]
other_columns =[
'PetID',
'AdoptionSpeed',
'Age',
'Color1',
'Color2',
'Color3',
'Dewormed',
'Fee',
'FurLength',
'Gender',
'Health',
'MaturitySize',
'PhotoAmt',
'Quantity',
'Sterilized',
'Vaccinated',
'VideoAmt',]<feature_engineering>
Age_map = {"Small children": 1, "Children": 2, "Teens": 3, "Adolescent": 4, "Young Adults": 5, "Adults": 6, "Elders": 7}
train["AgeGroups"] = train["AgeGroups"].map(Age_map)
 test["AgeGroups"] = test["AgeGroups"].map(Age_map)
 train.head()
categorical_columns = ['Gender', 'Color1', 'Color2', 'Color3','Vaccinated', 'Dewormed',
'Sterilized', 'State','Breed1_le', 'RescuerID']
freq_categorical_columns = []
for c in tqdm(categorical_columns):
df_all[c+'_freq'] = df_all[c].map(df_all.groupby(c ).size() / df_all.shape[0])
 freq_categorical_columns.append(c+'_freq')
 indexer = pd.factorize(df_all[c], sort=True)[1]
df_all[c] = indexer.get_indexer(df_all[c] )<set_options>
train = train.drop(["Name"], axis = 1)
 test = test.drop(["Name"], axis = 1)
 train = train.drop(["Age"], axis = 1)
 test = test.drop(["Age"], axis = 1 )
del train
del test
gc.collect()<define_variables>
sex_mapping = {"male": 0, "female": 1}
train['Sex'] = train['Sex'].map(sex_mapping)
 test['Sex'] = test['Sex'].map(sex_mapping)
 train.head()
excl_columns = list(df_all.dtypes[df_all.dtypes == 'object'].index.values )<features_selection>
embarked_mapping = {"S": 1, "C": 2, "Q": 3}
train['Embarked'] = train['Embarked'].map(embarked_mapping)
 test['Embarked'] = test['Embarked'].map(embarked_mapping)
 train.head()
df_all = df_all.loc[:, ~df_all.columns.isin(excl_columns)]<count_duplicates>
predict = train.drop(['Survived', 'PassengerId'], axis=1)
 target = train["Survived"]
x_train, x_val, y_train, y_val = train_test_split(predict, target, test_size = 0.22, random_state = 0 )
train = df_all.loc[df_all.is_train==1, ~df_all.columns.duplicated() ].drop('is_train', 1)
 test = df_all.loc[df_all.is_train==0, ~df_all.columns.duplicated() ].drop('is_train', 1 )<load_from_csv>
gaussian = GaussianNB()
gaussian.fit(x_train, y_train)
 y_pred = gaussian.predict(x_val)
 acc_gaussian = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gaussian )
<prepare_x_and_y>
logreg = LogisticRegression()
logreg.fit(x_train, y_train)
 y_pred = logreg.predict(x_val)
 acc_logreg = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_logreg )
feat_cols = list(train.columns[ 
(train.columns.values != 'AdoptionSpeed')
 ].values)
 y_col = 'AdoptionSpeed'<prepare_x_and_y>
svc = SVC()
svc.fit(x_train, y_train)
 y_pred = svc.predict(x_val)
 acc_svc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_svc )
X_train = train[feat_cols]
X_test = test[feat_cols]
y_train = train[y_col]<train_model>
linear_svc = LinearSVC()
linear_svc.fit(x_train, y_train)
 y_pred = linear_svc.predict(x_val)
 acc_linear_svc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_linear_svc )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _loss(self, coef, X, y, idx):
X_p = np.array([to_bins(pred, coef)for pred in X])
 ll = -cohen_kappa_score(y, X_p, weights='quadratic')
 return ll
def fit(self, X, y):
coef = [1.5, 2.0, 2.5, 3.0]
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [(1, 2),(1.8, 2.5),(2, 2.8),(2.5, 3.0)]
for it1 in range(10):
for idx in range(4):
a, b = ab_start[idx]
coef[idx] = a
la = self._loss(coef, X, y, idx)
 coef[idx] = b
lb = self._loss(coef, X, y, idx)
 for it in range(20):
if la > lb:
a = b -(b - a)* golden1
coef[idx] = a
la = self._loss(coef, X, y, idx)
 else:
b = b -(b - a)* golden2
coef[idx] = b
lb = self._loss(coef, X, y, idx)
 self.coef_ = {'x': coef}
def predict(self, X, coef):
X_p = np.array([to_bins(pred, coef)for pred in X])
 return X_p
def coefficients(self):
return self.coef_['x']
def to_bins(x, borders):
for i in range(len(borders)) :
if x <= borders[i]:
return i
return len(borders )<import_modules>
perceptron = Perceptron()
perceptron.fit(x_train, y_train)
 y_pred = perceptron.predict(x_val)
 acc_perceptron = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_perceptron )
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold
from bayes_opt import BayesianOptimization<init_hyperparams>
decisiontree = DecisionTreeClassifier()
decisiontree.fit(x_train, y_train)
 y_pred = decisiontree.predict(x_val)
 acc_decisiontree = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_decisiontree )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'subsample': 0.8,
'tree_method': 'gpu_hist',
'device': 'gpu',
'colsample_bytree': 0.8152926875727884,
'eta': 0.03402795645208036,
'gamma': 0.9209917577458885,
'max_depth': 6,
'silent': 1,
}<split>
randomforest = RandomForestClassifier()
randomforest.fit(x_train, y_train)
 y_pred = randomforest.predict(x_val)
 acc_randomforest = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_randomforest )
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
oof_coefficients = []
i = 0
for train_idx, valid_idx in kf.split(X_train, y_train.values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = y_train.values[train_idx]
y_val = y_train.values[valid_idx]
d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 optR = OptimizedRounder()
optR.fit(valid_pred, y_val)
 coefficients = optR.coefficients()
oof_coefficients.append(coefficients)
 pred_val_y_k = optR.predict(valid_pred, coefficients)
 qwk = cohen_kappa_score(y_val, pred_val_y_k, weights='quadratic')
 print("QWK = ", qwk)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test, oof_coefficients<compute_test_metric>
knn = KNeighborsClassifier()
knn.fit(x_train, y_train)
 y_pred = knn.predict(x_val)
 acc_knn = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_knn )
model, oof_train, oof_test, oof_coefficients = run_xgb(xgb_params, X_train, X_test )<compute_train_metric>
sgd = SGDClassifier()
sgd.fit(x_train, y_train)
 y_pred = sgd.predict(x_val)
 acc_sgd = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_sgd )
optR = OptimizedRounder()
mean_coeffs = np.mean(np.array(oof_coefficients), axis=0)
 valid_pred = optR.predict(oof_train, mean_coeffs)
 qwk = cohen_kappa_score(y_train.values, valid_pred, weights='quadratic')
 print("QWK = ", qwk )<choose_model_class>
gbk = GradientBoostingClassifier()
gbk.fit(x_train, y_train)
 y_pred = gbk.predict(x_val)
 acc_gbk = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gbk )
<import_modules>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron', 'Linear SVC',
'Decision Tree', 'Stochastic Gradient Descent', 'Gradient Boosting Classifier'],
'Score': [acc_svc, acc_knn, acc_logreg,
acc_randomforest, acc_gaussian, acc_perceptron,acc_linear_svc, acc_decisiontree,
acc_sgd, acc_gbk]})
 models.sort_values(by='Score', ascending=False )
<init_hyperparams><EOS> 
ids = test['PassengerId']
predictions = gbk.predict(test.drop('PassengerId', axis=1))
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )