Kaggle competition: Titanic - Machine Learning from Disaster
df_test_A = pd.read_csv(root/'Atest.dat',delim_whitespace=True)
 df_test_A['Datetime'] = pd.to_datetime(( df_test_A['YEAR']+1900 ).astype('str')\
+'-'+(df_test_A['MONTH'] ).astype('str')\
+'-'+(df_test_A['DAY'] ).astype('str')\
+' '+(df_test_A['HOUR']/100 ).astype('int' ).astype('str')\
+':00')
 df_test_A.set_index('Datetime', inplace=True)
 df_test_A = df_test_A.drop(['MONTH','DAY','YEAR','HOUR'],axis=1)
 df_test_A['hourOfDay'] = df_test_A.index.hour
df_test_A['dayOfWeek'] = df_test_A.index.dayofweek<load_from_csv>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 print("Train dataset has {} samples and {} attributes".format(*train.shape))
print("Test dataset has {} samples and {} attributes".format(*test.shape))
df_test_B = pd.read_csv(root/'Btest.dat',header=None,delim_whitespace=True)\ 
.rename(columns={0:'Date',1:'HorizRad',2:'SE_Rad',3:'S_Rad',4:'SW_Rad'})
 df_test_B.set_index('Date', inplace=True)
 df_test_B.reset_index(inplace=True)
 df_test_B['Day'] = np.floor(df_test_B['Date'])
 df_test_B['Time'] = df_test_B['Date'] - df_test_B['Day']
df_test_B.drop('Date', axis=1, inplace=True )<data_type_conversions>
n=len(train)
 surv_0=len(train[train['Survived']==0])
 surv_1=len(train[train['Survived']==1])
 print("% of passanger survived in train dataset: ",surv_1*100/n)
 print("% of passanger not survived in train dataset: ",surv_0*100/n )
df_train_A = pd.read_csv(root/'Atrain.dat',delim_whitespace=True)
 df_train_A['Datetime'] = pd.to_datetime(( df_train_A['YEAR']+1900 ).astype('str')\
+'-'+(df_train_A['MONTH'] ).astype('str')\
+'-'+(df_train_A['DAY'] ).astype('str')\
+' '+(df_train_A['HOUR']/100 ).astype('int' ).astype('str')\
+':00')
 df_train_A.set_index('Datetime', inplace=True)
 df_train_A = df_train_A.drop(['MONTH','DAY','YEAR','HOUR'],axis=1)
 df_train_A['hourOfDay'] = df_train_A.index.hour
df_train_A['dayOfWeek'] = df_train_A.index.dayofweek
df_train_A<feature_engineering>
cat=['Pclass','Sex','Embarked']
num=['Age','SibSp','Parch','Fare']
df_train_B = pd.read_csv(root/'Btrain.dat',header=None,delim_whitespace=True)\ 
.rename(columns={0:'Date',1:'HorizRad',2:'SE_Rad',3:'S_Rad',4:'SW_Rad',5:'Beam_Rad'})
 df_train_B.set_index('Date', inplace=True)
 df_train_B.sort_index(inplace=True)
 df_train_B.reset_index(inplace=True)
 df_train_B['Day'] = np.floor(df_train_B['Date'])
 df_train_B['Time'] = df_train_B['Date'] - df_train_B['Day']
df_train_B.drop('Date', axis=1, inplace=True)
 df_train_B<concatenate>
csq=chi2_contingency(pd.crosstab(train['Survived'], train['Sex']))
print("P-value: ",csq[1] )
df_weather_merged = df_train_A[['TEMP','SOLAR']].append(df_test_A[['TEMP','SOLAR']])
 df_weather_merged = df_weather_merged.sort_index()
df_weather_merged<define_variables>
csq2=chi2_contingency(pd.crosstab(train['Survived'], train['Embarked']))
print("P-value: ",csq2[1] )
alpha_data = {}
MAPE_data = {}
RSQUARED_data = {}
NMBE_data = {}
CVRSME_data = {}
data_test_A = df_test_A.copy()
data_test_B = df_test_B.copy()<merge>
csq3=chi2_contingency(pd.crosstab(train['Survived'], train['Pclass']))
print("P-value: ",csq3[1] )
for name_meter in ['WBE','WBCW','WBHW']:
data_train_A = df_train_A.copy()
data_train_A = data_train_A.merge(df_weather_merged, left_index=True, right_index=True)
 feat_train_A = data_train_A.drop(['WBE','WBCW','WBHW'],axis=1 ).copy()
label_train_A = data_train_A[name_meter].copy()
feat_test_A = df_test_A.copy()
feat_test_A = feat_test_A.merge(df_weather_merged, left_index=True, right_index=True)
 print('Power Meter: '+name_meter)
 LGB_model = LGBMRegressor()
data_train_A[name_meter + '_pred'] = cross_val_predict(LGB_model, feat_train_A, label_train_A, cv=5)
 LGB_model.fit(feat_train_A, label_train_A)
 data_test_A[name_meter + '_pred'] = LGB_model.predict(feat_test_A)
 errors = abs(data_train_A[name_meter + '_pred'] - label_train_A)
 errors_mean = round(np.mean(errors), 2)
 MAPE = 100 * np.mean(( errors / label_train_A))
NMBE = 100 *(sum(label_train_A - data_train_A[name_meter + '_pred'])/(pd.Series(label_train_A ).count() * np.mean(label_train_A)))
 CVRSME = 100 *(( sum(( label_train_A - data_train_A[name_meter + '_pred'])**2)/(pd.Series(label_train_A ).count() -1)) **(0.5)) / np.mean(label_train_A)
 RSQUARED = r2_score(label_train_A, data_train_A[name_meter + '_pred'])
 print("MAPE: "+str(round(MAPE,2)))
 print("NMBE: "+str(round(NMBE,2)))
 print("CVRSME: "+str(round(CVRSME,2)))
 print("R SQUARED: "+str(round(RSQUARED,2)))
 data_train_A[[name_meter,name_meter + '_pred']].iplot(kind='scatter', filename='cufflinks/cf-simple-line')
 print('-----------------------------------------------------------' )<find_best_model_class>
print(train.isnull().sum() )
name_meter = 'Beam_Rad'
data_train_B = df_train_B.copy()
feat_train_B = data_train_B.drop('Beam_Rad',axis=1 ).copy()
label_train_B = data_train_B[name_meter].copy()
feat_test_B = df_test_B.copy()
print('Power Meter: '+name_meter)
 LGB_model = LGBMRegressor()
data_train_B[name_meter + '_pred'] = cross_val_predict(LGB_model, feat_train_B, label_train_B, cv=5)
 LGB_model.fit(feat_train_B, label_train_B)
 data_test_B[name_meter + '_pred'] = LGB_model.predict(feat_test_B)
 errors = abs(data_train_B[name_meter + '_pred'] - label_train_B)
 errors_mean = round(np.mean(errors), 2)
 MAPE = 100 * np.mean(( errors / label_train_B))
NMBE = 100 *(sum(label_train_B - data_train_B[name_meter + '_pred'])/(pd.Series(label_train_B ).count() * np.mean(label_train_B)))
 CVRSME = 100 *(( sum(( label_train_B - data_train_B[name_meter + '_pred'])**2)/(pd.Series(label_train_B ).count() -1)) **(0.5)) / np.mean(label_train_B)
 RSQUARED = r2_score(label_train_B, data_train_B[name_meter + '_pred'])
 print("MAPE: "+str(round(MAPE,2)))
 print("NMBE: "+str(round(NMBE,2)))
 print("CVRSME: "+str(round(CVRSME,2)))
 print("R SQUARED: "+str(round(RSQUARED,2)))
 data_train_B[[name_meter,name_meter + '_pred']].iplot(kind='scatter', filename='cufflinks/cf-simple-line')
 print('-----------------------------------------------------------' )<feature_engineering>
print(test.isnull().sum() )
df_submit.loc[df_submit['row_id'].str.endswith('WBE'), 'target'] = data_test_A[['WBE_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('WBCW'), 'target'] = data_test_A[['WBCW_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('WBHW'), 'target'] = data_test_A[['WBHW_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('true_beam_insolation'), 'target'] = data_test_B['Beam_Rad_pred'].values
df_submit<save_to_csv>
med=np.nanmedian(train['Age'])
 train['Age']=train['Age'].fillna(med)
 test['Age']=test['Age'].fillna(med )
df_submit.to_csv('submission.csv', index=False )<create_dataframe>
train['Cabin'].value_counts()
modelling_data_feat = np.load('.. /input/x_train.npy')
 modelling_data_target = np.load('.. /input/y_train.npy')
 submit_data_feat = np.load('.. /input/x_test.npy')
 df_mod_feat = pd.DataFrame(data=modelling_data_feat)
 df_sub_feat = pd.DataFrame(data=submit_data_feat)
 rs = 4319<categorify>
train['Cabin']=train['Cabin'].fillna(0)
 test['Cabin']=test['Cabin'].fillna(0 )
df_mod_feat = df_mod_feat.drop(columns=['waterfront','lat','long'])
 df_sub_feat = df_sub_feat.drop(columns=['waterfront','lat','long'])
 df_mod_feat_cat = df_mod_feat[['zipcode']].copy()
df_mod_feat_num = df_mod_feat.drop(columns=['zipcode'])
 df_sub_feat_cat = df_sub_feat[['zipcode']].copy()
df_sub_feat_num = df_sub_feat.drop(columns=['zipcode'])
 cat_allfeat = pd.concat([df_mod_feat_cat, df_sub_feat_cat])
 enc = OneHotEncoder()
temp = enc.fit_transform(cat_allfeat)
 temp = pd.DataFrame(temp.todense())
 df_mod_feat_cat_conv = temp.iloc[:df_mod_feat_cat.shape[0]]
df_sub_feat_cat_conv = temp.iloc[df_mod_feat_cat.shape[0]:cat_allfeat.shape[0]]
df_sub_feat_cat_conv = df_sub_feat_cat_conv.reset_index(drop=True)
 for i in range(0, df_mod_feat_num.shape[0]):
df_mod_feat_num.loc[i,'date'] = int(df_mod_feat_num.loc[i,'date'][0:4])
 if df_mod_feat_num.loc[i,'yr_renovated'] == 0:
df_mod_feat_num.loc[i,'yr_renovated'] = df_mod_feat_num.loc[i,'yr_built']
for i in range(0, df_sub_feat_num.shape[0]):
df_sub_feat_num.loc[i,'date'] = int(df_sub_feat_num.loc[i,'date'][0:4])
 if df_sub_feat_num.loc[i,'yr_renovated'] == 0:
df_sub_feat_num.loc[i,'yr_renovated'] = df_sub_feat_num.loc[i,'yr_built']
df_mod_feat = pd.concat([df_mod_feat_cat_conv, df_mod_feat_num], axis=1)
 df_sub_feat = pd.concat([df_sub_feat_cat_conv, df_sub_feat_num], axis=1)
<split>
train['Embarked'].value_counts()
def mean_absolute_percentage_error(y_true, y_pred):
return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100
X_train, X_test, y_train, y_test = train_test_split(df_mod_feat, modelling_data_target, test_size=0.1, random_state=rs)
 model = LinearRegression(fit_intercept=False)
 selector = RFECV(estimator=model, step=1, min_features_to_select=1, cv=StratifiedKFold(n_splits=5, random_state=rs),scoring='r2')
 feat_res = selector.fit(X_train, y_train)
 print("Optimal number of features : %d" % selector.n_features_)
 sel_X_train = feat_res.transform(X_train)
 sel_X_test = feat_res.transform(X_test)
 y_true, y_pred = y_test, feat_res.estimator_.predict(sel_X_test)
 mean_absolute_percentage_error(y_true, y_pred )<save_to_csv>
train['Cabin']=train['Cabin'].fillna("S" )
submission_features = feat_res.transform(df_sub_feat)
 test_predictions = feat_res.estimator_.predict(submission_features)
 submission = pd.DataFrame({'Id': range(1, test_predictions.shape[0]+1), 'Price': test_predictions})
 submission = submission.reset_index(drop=True)
 submission.to_csv('submission.csv', index=False)
 submission<define_variables>
med=np.nanmedian(train['Fare'])
 test['Fare']=test['Fare'].fillna(med )
%matplotlib inline
data_dir = '.. /input/'<load_from_csv>
train['hasCabin']=train['Cabin'].apply(lambda x: 0 if x==0 else 1)
 test['hasCabin']=test['Cabin'].apply(lambda x: 0 if x==0 else 1 )
def categories_to_indicators(in_df):
new_df = in_df.copy()
new_df['IsMale'] = in_df['PatientSex'].map(lambda x: 'M' in x ).astype(float)
 new_df['IsAP'] = in_df['ViewPosition'].map(lambda x: 'AP' in x ).astype(float)
 return new_df.drop(['PatientSex', 'ViewPosition'], axis=1)
 full_train_df = categories_to_indicators(pd.read_csv(os.path.join(data_dir, 'train_all.csv')))
 full_stack = imread(os.path.join(data_dir, 'train.tif'))
full_train_df['image'] = full_train_df['slice_idx'].map(lambda x: full_stack[x])
 full_train_df.sample(3 )<load_from_csv>
train['FamilyMem']=train.apply(lambda x: x['SibSp']+x['Parch'], axis=1)
 test['FamilyMem']=test.apply(lambda x: x['SibSp']+x['Parch'], axis=1 )
submission_test_df = categories_to_indicators(pd.read_csv(os.path.join(data_dir, 'test_info.csv')))
 test_stack = imread(os.path.join(data_dir, 'test.tif'))
submission_test_df['image'] = submission_test_df['slice_idx'].map(lambda x: full_stack[x])
 submission_test_df.sample(3 )<train_model>
train['title']=train['Name'].apply(get_title)
 test['title']=test['Name'].apply(get_title )
def fit_and_score(in_model, feature_maker, rescale=True):

 full_features = feature_maker(full_train_df)
 full_labels = full_train_df['opacity']
submission_feat = feature_maker(submission_test_df)
 train_feat, valid_feat, train_lab, valid_lab = train_test_split(full_features,
full_labels,
test_size=0.25,
random_state=2018)
 if rescale:
feature_scaler = RobustScaler()
train_feat = feature_scaler.fit_transform(train_feat)
 valid_feat = feature_scaler.transform(valid_feat)
 submission_feat = feature_scaler.transform(submission_feat)
 in_model.fit(train_feat, train_lab)
 predictions = in_model.predict_proba(valid_feat)[:, 1]
predicted_class = predictions>0.5
tpr, fpr, _ = roc_curve(valid_lab, predictions)
 auc = roc_auc_score(valid_lab, predictions)
 fig,(ax1, ax2)= plt.subplots(1, 2, figsize=(20, 10))
ax1.plot(tpr, fpr, 'r.-', label='Prediction(AUC:{:2.2f})'.format(auc))
ax1.plot(tpr, tpr, 'k-', label='Random Guessing')
 ax1.legend()
ax1.set_title('ROC Curve')
 print(classification_report(valid_lab, predicted_class, target_names=['opacity', 'no opacity']))
sns.heatmap(confusion_matrix(valid_lab, predicted_class),
annot=True, fmt='4d', ax=ax2)
 ax2.set_xlabel('Prediction')
 ax2.set_ylabel('Actual Value')
 ax2.set_title('Confusion Matrix({:.1%})'.format(accuracy_score(valid_lab, predicted_class)))
 sub_df = submission_test_df[['tile_id']].copy()
sub_df['opacity'] = in_model.predict_proba(submission_feat)[:, 1]
sub_df[['tile_id', 'opacity']].to_csv(
 'm-{model}-f-{features}-s-{auc:2.0f}.csv'.format(
 model=in_model.__class__.__name__,
features=feature_maker.__name__,
auc=100*auc, 
), index=False )<train_model>
title_lev1=list(train['title'].value_counts().reset_index() ['index'])
 title_lev2=list(test['title'].value_counts().reset_index() ['index'] )
dum_model = DummyClassifier(strategy='stratified', random_state=2018)
 def justage(in_df):
return in_df[['PatientAge']].values
fit_and_score(
 dum_model,
justage
)<choose_model_class>
title_lev=list(set().union(title_lev1, title_lev2))
print(title_lev )
knn_model = KNeighborsClassifier(1)
 def table_features(in_df):
return in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].values
fit_and_score(
 knn_model,
table_features
)<train_model>
train['title']=pd.Categorical(train['title'], categories=title_lev)
 test['title']=pd.Categorical(test['title'], categories=title_lev )
def basic_image_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
out_df['Mean_Intensity'] = in_df['image'].map(np.mean)
 out_df['Std_Intensity'] = in_df['image'].map(np.std)
 return out_df.values
knn_model = KNeighborsClassifier(2)
 fit_and_score(
 knn_model,
basic_image_features
)<train_model>
cols=['Pclass','Sex','Embarked','hasCabin','title']
fcol=['Pclass','Sex','Embarked','hasCabin','title','Age','FamilyMem','Fare']
lr_model = LogisticRegression(solver='lbfgs', random_state=2018)
 fit_and_score(
 lr_model,
basic_image_features
)<train_model>
for c in cols:
train[c]=train[c].astype('category')
 test[c]=test[c].astype('category' )
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
basic_image_features
)<train_model>
train_df=train[fcol]
test_df=test[fcol]
nb_model = GaussianNB()
fit_and_score(
 nb_model,
basic_image_features
)<train_model>
train_df=pd.get_dummies(train_df, columns=cols, drop_first=True)
 test_df=pd.get_dummies(test_df, columns=cols, drop_first=True )
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
basic_image_features
)<train_on_grid>
y=train['Survived']
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
texture_features
)<prepare_x_and_y>
x_train, x_test, y_train, y_test = train_test_split(train_df, y, test_size=0.3, random_state=42 )
c_model = models.Sequential()
c_model.add(PTModel(include_top=False,
input_shape=full_train_df['image'].iloc[0].shape+(3,),
weights='imagenet'))
c_model.add(layers.GlobalAvgPool2D())
 def vgg_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
full_image_stack = np.stack(in_df['image'], 0)
 color_image_stack = np.stack([full_image_stack, full_image_stack, full_image_stack], -1 ).astype(float)
 pp_color_image_stack = preprocess_input(color_image_stack)
 vgg_features = c_model.predict(pp_color_image_stack)
 return np.concatenate([out_df.values, vgg_features], 1 )<train_model>
rfc=RandomForestClassifier(random_state=42 )
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
vgg_features
)<train_model>
param_grid = {
'n_estimators': [200, 500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [4,5,6,7,8],
'criterion' :['gini', 'entropy']
}
xg_model = XGBClassifier()
fit_and_score(
 xg_model,
vgg_features
)<split>
CV_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)
 CV_rfc.fit(x_train, y_train )
trn_image, vld_image, trn_label , vld_label = train_test_split(full_train_df['image'],
full_train_df['opacity'],
test_size=0.25,
random_state=2018)
 trn_image = np.stack(trn_image, 0)
 vld_image = np.stack(vld_image, 0 )<choose_model_class>
CV_rfc.best_params_
out_model = models.Sequential()
out_model.add(layers.Reshape(( 64, 64, 1), input_shape=trn_image.shape[1:]))
out_model.add(layers.Conv2D(16,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(32,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(64,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(128,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.GlobalAveragePooling2D())
 out_model.add(layers.Dense(32, activation='relu'))
out_model.add(layers.Dense(1, activation='sigmoid'))
out_model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['binary_accuracy'])
 out_model.summary()<train_model>
rfc1=RandomForestClassifier(random_state=42, max_features='auto', n_estimators= 200, max_depth=8, criterion='gini' )
fit_results = out_model.fit(trn_image, trn_label,
validation_data=(vld_image, vld_label),
epochs=100)
 clear_output()<predict_on_test>
rfc1.fit(x_train, y_train )
def custom_cnn_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
full_image_stack = np.stack(in_df['image'], 0)
 model_pred = out_model.predict(full_image_stack)
 return np.concatenate([out_df.values, model_pred], 1 )<train_model>
pred=rfc1.predict(x_test )
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
custom_cnn_features
)<train_model>
print("Accuracy for Random Forest on CV data: ",accuracy_score(y_test,pred))
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
custom_cnn_features
)<train_model>
op_rf=rfc1.predict(test_df )
xg_model = XGBClassifier()
fit_and_score(
 xg_model,
custom_cnn_features
)<load_from_csv>
op=pd.DataFrame(test['PassengerId'])
 op['Survived']=op_rf
<set_options><EOS> 
op.to_csv("Submission.csv", index=False )