Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
warnings.filterwarnings('ignore')
 %matplotlib inline
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, bootstrap=True, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
preds = np.stack([t.predict(X_valid)for t in m.estimators_])
<import_modules>
passenger_id = test['PassengerId']
target = train['Survived']
from sklearn import *<compute_test_metric>
for df in [train,test]:
df.drop('PassengerId',axis=1,inplace=True )
plt.plot([metrics.r2_score(y_valid, np.mean(preds[:i+1], axis=0)) for i in range(1000)]);<choose_model_class>
all_data = pd.concat([train.drop('Survived',axis=1),test] ).reset_index(drop=True )
??RandomForestRegressor<train_model>
ntrain = train.shape[0]
ntest = test.shape[0]
m = RandomForestRegressor(bootstrap=True, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )<set_options>
all_data['FamSize'] = 1 + all_data['SibSp'] + all_data['Parch']
all_data.drop(['SibSp','Parch'],axis=1,inplace=True )
set_rf_samples(50000 )<split>
total_miss = all_data.isnull().sum()
percent_miss =(total_miss/all_data.isnull().count() *100)
 missing_data = pd.DataFrame({'Total missing':total_miss,'% missing':percent_miss})
 missing_data.sort_values(by='Total missing',ascending=False ).head()
def split_vals(a,n): return a[:n].copy() , a[n:].copy()
n_valid = int(len(processed)*0.2)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(processed, n_trn)
 X_train, X_valid = split_vals(X, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train.shape, y_train.shape, X_valid.shape<split>
all_data[all_data['Embarked'].isnull() ]
raw_train, raw_valid = split_vals(df, n_trn )<predict_on_test>
display(all_data[all_data['Fare'].isnull() ])
 display(all_data[all_data['Fare']==0] )
def get_preds(t): return t.predict(X_valid)
 %time preds = np.stack(parallel_trees(m, get_preds))
np.mean(preds[:,0]), np.std(preds[:,0] )<feature_engineering>
_ = all_data.set_value(1043,'Fare',value=0 )
x['pred_std'] = np.std(preds, axis=0)
 x['pred'] = np.mean(preds, axis=0)
 flds = ['Neighborhood', 'SalePrice', 'pred', 'pred_std']
nei_summ = x[flds].groupby('Neighborhood', as_index=False ).mean()
nei_summ.head()<train_model>
splits = 5
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=100, bootstrap=True, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<compute_test_metric>
def discretize_fare(val):
fare_group = pd.qcut(all_data['Fare'],splits ).sort_values().unique()
for i in range(splits):
if val in fare_group[i]:
return i+1
elif np.isnan(val):
return val
fi = rf_feat_importance(m, X )<compute_test_metric>
all_data['Fare'] = all_data['Fare'].apply(discretize_fare )
fi = rf_feat_importance(m, df_trn )<split>
all_data['Fare'] = all_data['Fare'].fillna(5 ).astype(int )
df_keep = df_trn[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )<create_dataframe>
def discretize_age(val):
age_group = pd.cut(all_data['Age'],splits ).sort_values().unique()
for i in range(splits):
if val in age_group[i]:
return i+1
elif np.isnan(val):
return 0
df_keep = X[to_keep].copy()<train_model>
all_data['Age'] = all_data['Age'].apply(discretize_age ).astype(int )
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, random_state=0)
 m.fit(df_keep, y )<train_model>
all_data['Title'] = all_data['Name'].apply(get_title )
m = RandomForestRegressor(max_features=0.5,
n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<count_values>
all_data['Title'] = all_data['Title'].replace(['Ms','Mlle'],'Miss')
 all_data['Title'] = all_data['Title'].replace('Mme','Mrs')
 all_data['Title'] = all_data['Title'].replace(['Don','Dona','Lady','Sir',
'Countess','Jonkheer'],'Royal')
 all_data['Title'] = all_data['Title'].replace(['Rev','Major','Col','Capt','Dr'],'Other' )
df.Neighborhood.value_counts()<count_values>
def impute_age(row):
pclass = row['Pclass']
title = row['Title']
age = row['Age']
if age == 0:
return int(round(all_data.loc[(all_data['Age']!=0)& 
(all_data['Pclass']==pclass)& 
(all_data['Title']==title)]['Age'].mean() ,1))
else:
return age
df.MSSubClass.value_counts()<count_values>
all_data['Age'] = all_data.apply(impute_age,axis=1 )
df.GarageType.value_counts()<import_modules>
_ = all_data.rename({'Cabin':'Deck'},axis=1,inplace=True )
from scipy.cluster import hierarchy as hc<import_modules>
all_data['Deck'] = all_data['Deck'].fillna('N' )
from scipy.stats import spearmanr as sp<train_model>
def cabin_to_deck(row):
return row['Deck'][0]
def get_oob(df):
m = RandomForestRegressor(n_estimators=500, n_jobs=-1, oob_score=True)
 x, _ = split_vals(df, n_trn)
 m.fit(x, y_train)
 return m.oob_score_<feature_engineering>
all_data['Deck'] = all_data.apply(cabin_to_deck,axis=1 )
get_oob(df_keep )<install_modules>
ticket_list = []
for ticket_id in list(all_data['Ticket'].unique()):
count = all_data[all_data['Ticket']==ticket_id].count() [0]
decks = all_data[all_data['Ticket']==ticket_id]['Deck']
empty_decks =(decks=='N' ).sum()
if(count > 1)and(empty_decks > 0)and(empty_decks < len(decks)) :
ticket_list.append(ticket_id)
 print(ticket_list )
<install_modules>
for ticket in ticket_list:
display(all_data[all_data['Ticket']==ticket] )
conda install -c conda-forge hdbscan<train_on_grid>
_ = all_data.set_value(533,'Deck',value=all_data.loc[128]['Deck'])
 _ = all_data.set_value(1308,'Deck',value=all_data.loc[128]['Deck'])
 _ = all_data.set_value(258,'Deck',all_data.loc[679]['Deck'])
 _ = all_data.set_value(373,'Deck',value='C')
 _ = all_data.set_value(290,'Deck',value=all_data.loc[741]['Deck'])
 _ = all_data.set_value(708,'Deck',value=all_data.loc[297]['Deck'])
 _ = all_data.set_value(1032,'Deck',value=all_data.loc[297]['Deck'])
 _ = all_data.set_value(306,'Deck',value='C')
 _ = all_data.set_value(1266,'Deck',value=all_data.loc[1033]['Deck'])
 _ = all_data.set_value(856,'Deck',value=all_data.loc[318]['Deck'])
 _ = all_data.set_value(1108,'Deck',value=all_data.loc[318]['Deck'])
 _ = all_data.set_value(380,'Deck',value='C')
 _ = all_data.set_value(557,'Deck',value='C')
 _ = all_data.set_value(537,'Deck',value='C')
 _ = all_data.set_value(1215,'Deck',value='E')
 _ = all_data.set_value(841,'Deck',value=all_data.loc[772]['Deck'] )
data = df_keep
clusterer = hdbscan.HDBSCAN(min_cluster_size=3, gen_min_span_tree=True)
 clusterer.fit(data )<import_modules>
decks_by_class = [[],[],[]]
for i in range(3):
decks_by_class[i] = list(all_data[all_data['Pclass']==i+1]['Deck'].unique())
 print(f'Pclass = {i+1} decks:',decks_by_class[i] )
from pdpbox import pdp
from plotnine import *<feature_engineering>
for i in range(3):
if 'N' in decks_by_class[i]:
decks_by_class[i].remove('N')
 if 'T' in decks_by_class[i]:
decks_by_class[i].remove('T' )
df_all = df_keep
df_all["SalePrice"] = y<install_modules>
weights_by_class = [[],[],[]]
for i,deck_list in enumerate(decks_by_class):
for deck in deck_list:
if i == 0:
class_total = all_data[(all_data['Deck']!='N')&(all_data['Pclass']==i+1)].count() [0]-1
else:
class_total = all_data[(all_data['Deck']!='N')&(all_data['Pclass']==i+1)].count() [0]
deck_total = all_data[(all_data['Deck']==deck)&(all_data['Pclass']==i+1)].count() [0]
weights_by_class[i].append(deck_total/class_total)
 print(f'Pclass = {i+1} weights:',np.round(weights_by_class[i],3))
!pip install scikit-misc<install_modules>
ticket_dict = {}
!pip install treeinterpreter<import_modules>
def impute_deck(row):
ticket = row['Ticket']
deck = row['Deck']
pclass = row['Pclass']
if(deck == 'N')and(ticket not in ticket_dict):
if pclass == 1:
deck = list(np.random.choice(decks_by_class[0],size=1,
p=weights_by_class[0])) [0]
elif pclass ==2:
deck = list(np.random.choice(decks_by_class[1],size=1,
p=weights_by_class[1])) [0]
elif pclass ==3:
deck = list(np.random.choice(decks_by_class[2],size=1,
p=weights_by_class[2])) [0]
ticket_dict[ticket] = deck
elif(deck == 'N')and(ticket in ticket_dict):
deck = ticket_dict[ticket]
return deck
from treeinterpreter import treeinterpreter as ti<split>
all_data['Deck'] = all_data.apply(impute_deck,axis=1 )
df_train, df_valid = split_vals(df[df_keep.columns], n_trn )<predict_on_test>
all_data = all_data.drop(['Name','Ticket','Title'],axis=1 )
prediction, bias, contributions = ti.predict(m, row )<sort_values>
all_data['Deck'] = all_data['Deck'].map({'F':0,'C':1,'E':2,
'G':3,'D':4,'A':5,
'B':6,'T':7} ).astype(int )
idxs = np.argsort(contributions[0] )<define_variables>
all_data['Embarked'] = all_data['Embarked'].map({'S':0,'C':1,'Q':2} ).astype(int )
[o for o in zip(df_keep.columns[idxs], df_valid.iloc[0][idxs], contributions[0][idxs])]<define_variables>
all_data['Sex'] = all_data['Sex'].map({'female':0,'male':1} ).astype(int )
columns = ['OverallQual', 'GrLivArea', 'YearBuilt', 'GarageCars',
'TotalBsmtSF', '1stFlrSF', 'BsmtFinSF1', 'GarageArea', 'LotArea']
df.loc[:, columns]<set_options>
all_data['Alone'] = 0
all_data.loc[all_data['FamSize']==1,'Alone'] = 1
%whos<define_variables>
from sklearn.model_selection import KFold, GridSearchCV, cross_val_score
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,ExtraTreesClassifier
from sklearn.svm import SVC
numeric_feats = X.dtypes[X.dtypes != "bool"].index<sort_values>
def rmse_cv(model,train):
kf = KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(train)
 return np.sqrt(-cross_val_score(model,train,target,scoring='neg_mean_squared_error',cv=kf))
def logloss_cv(model,train):
kf = KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(train)
 return -cross_val_score(model,train,target,scoring='neg_log_loss',cv=kf)
 def accuracy_cv(model,train):
kf = KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(train.values)
 return cross_val_score(model,train,target,scoring='accuracy',cv=kf )
skewed_feats = X.apply(lambda x: skew().sort_value<feature_engineering>
def rmse(y_true,y_pred):
return np.sqrt(mean_squared_error(y_true,y_pred))
def accuracy(y_true,y_pred):
return accuracy_score(y_true,y_pred )
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )<categorify>
rf = RandomForestClassifier(n_estimators=700,max_depth=4,
min_samples_leaf=1,n_jobs=-1,
warm_start=True,
random_state=42)
 et = ExtraTreesClassifier(n_estimators=550,max_depth=4,
min_samples_leaf=1,n_jobs=-1,
random_state=42)
 ada = AdaBoostClassifier(n_estimators=550,learning_rate=0.001,
random_state=42)
 svc = SVC(C=2,probability=True,random_state=42 )
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape<define_variables>
from sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin, clone
from sklearn.metrics import mean_squared_error, accuracy_score
VAL_FILE = '.. /input/training-and-validation-data-pickle/validation.pkl.gz'<define_variables>
class StackerLvl1(BaseEstimator, ClassifierMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def oof_pred(self, X, y):
self.base_models_ = [list() for x in self.base_models]
kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, test_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X.loc[train_index], y.loc[train_index])
 y_pred = instance.predict(X.loc[test_index])
 out_of_fold_predictions[test_index, i] = y_pred
return out_of_fold_predictions
def fit(self, X, y):
self.meta_model_ = clone(self.meta_model)
 self.meta_model_.fit(self.oof_pred(X,y), y)
 return self
def predict(self, test):
self.meta_features_ = np.column_stack([
np.column_stack([model.predict(test)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(self.meta_features_ )
almost_zero = 1e-10
almost_one = 1 - almost_zero<define_variables>
train = all_data[:ntrain]
test = all_data[ntrain:]
base_models = {
'lgb1 ': "Python LGBM based on Pranav Pandya's R version",
'wbftl': "anttip's Wordbatch FM-FTRL",
'nngpu': "Downampled Neural Network run on GPU"
}<define_variables>
stack_model = StackerLvl1(base_models=(rf,et,svc),meta_model = ada)
 stack_model.fit(train,target)
 print('Accuracy:',accuracy(stack_model.predict(train),target))
print('RMSE:',rmse(stack_model.predict(train),target))
cvfiles = {
'lgb1 ': '.. /input/validate-pranav-lgb-model/pranav_lgb_val_nostop.csv',
'wbftl': '.. /input/validate-anttip-s-wordbatch-fm-ftrl-9711-version/wordbatch_fm_ftrl_val.csv',
'nngpu': '.. /input/gpu-validation/gpu_val1.csv'
}<define_variables>
stack_model_pred = stack_model.predict(test )
<define_variables><EOS> 
sub = pd.DataFrame({'PassengerId':passenger_id,
'Survived':stack_model_pred})
 sub.to_csv('submission.csv',index=False )