Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_missing_values>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
print(leak_df.isnull().sum() )<set_options>
train = pd.read_csv('.. /input/titanic/train.csv')
 train.head()
gc.collect()<load_from_csv>
categorical = ['Pclass', 'Sex', 'Embarked']
numerical = ['Age', 'Fare', 'SibSp', 'Parch']
target = 'Survived'
kfold_submission = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 halfhalf_submission = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 highway_submission = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0)
 gep3_submission = pd.read_csv(".. /input/ashrae-great-energy-predictor-iii-eda-model/Normal.csv", index_col=0)
 simple_submission = pd.read_csv(".. /input/ashrae-simple-data-cleanup-lb-1-08-no-leaks/submission.csv", index_col=0 )<feature_engineering>
train['Fare'] = np.log1p(train['Fare'])
 train['Age'] = np.log1p(train['Age'] )
test_df["pred1"] = kfold_submission.meter_reading
test_df["pred2"] = halfhalf_submission.meter_reading
test_df["pred3"] = highway_submission.meter_reading
test_df["pred4"] = gep3_submission.meter_reading
test_df["pred5"] = simple_submission.meter_reading
del kfold_submission, halfhalf_submission, highway_submission, gep3_submission
del simple_submission
gc.collect()<feature_engineering>
mask =(train[target] == 1)
 t_result = []
for num in numerical:
t_stat, p_val = ttest_ind(( train[mask])[num], 
(train[~mask])[num],
equal_var = True,
nan_policy = 'omit')
 t_result.append({
'group_1' : 'Survived',
'group_2' : 'Died',
'variable' : num,
't_stat' : t_stat,
'p_value' : p_val
})
 t_result = pd.DataFrame(t_result )
test_df.loc[test_df.pred3 < 0, "pred3"] = 0
test_df.loc[test_df.pred4 < 0, "pred4"] = 0
test_df.loc[test_df.pred5 < 0, "pred5"] = 0
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
def get_titles(data):
title_re = re.compile(r'(?:^.+),(\w+)')
 titles = []
for name in data['Name']:
titles.append(title_re.findall(name)[0])
 data['Title'] = titles
for i, title in enumerate(data['Title']):
if title in ['Miss', 'Ms', 'Mlle', 'Mrs', 'Mme']:
cat = 'Mrs_Ms'
elif title in ['Mr', 'Don']:
cat = 'Mr'
elif title in ['Master']:
cat = 'Master'
else:
cat = 'Honorable'
data.loc[i, 'Title'] = cat
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'pred4', 'pred5', 'row_id']],
left_on = ['building_id','meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'],
how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
chi2_result = []
for cat in categorical:
crosstab = pd.crosstab(train[cat], train[target])
 chi2_stat, p_val, dof, ex = chi2_contingency(crosstab)
 chi2_result.append({
'var_1' : cat,
'var_2' : target,
'chi2' : chi2_stat,
'dof' : dof,
'p_value' : p_val
})
 chi2_result = pd.DataFrame(chi2_result )
leak_df["pred1_l1p"] = np.log1p(leak_df.pred1)
 leak_df["pred2_l1p"] = np.log1p(leak_df.pred2)
 leak_df["pred3_l1p"] = np.log1p(leak_df.pred3)
 leak_df["pred4_l1p"] = np.log1p(leak_df.pred4)
 leak_df["pred5_l1p"] = np.log1p(leak_df.pred5)
 leak_df["meter_reading_l1p"] = np.log1p(leak_df.meter_reading )<count_missing_values>
train.isnull().sum()
leak_df.isnull().sum()<split>
train.isnull().sum()
print(leak_df[leak_df.pred1_l1p.isnull() ])
 print(leak_df[leak_df.pred4_l1p.isnull() ])
 print(leak_df[leak_df.pred5_l1p.isnull() ] )<set_options>
X = train.copy()
y = X.pop(target )
del building_meta_df
gc.collect()<define_search_space>
median_imputer = SimpleImputer(strategy = 'median')
 mode_imputer = SimpleImputer(strategy = 'most_frequent')
 missing_transformer = ColumnTransformer([('num', median_imputer, numerical), 
('cat', mode_imputer, categorical)])
 missing_transformer.fit(X)
 X_imp = pd.DataFrame(missing_transformer.transform(X))
X_imp.columns = numerical + categorical
X.drop(numerical + categorical, axis = 1, inplace = True)
 X = pd.concat([X, X_imp], axis = 1 )
all_combinations = list(np.linspace(0.15,0.4,15))
all_combinations<import_modules>
ordinal_encoder = OrdinalEncoder()
X_enc = pd.DataFrame(ordinal_encoder.fit_transform(X[categorical]))
X_enc.columns = categorical
X.drop(categorical, axis = 1, inplace = True)
 X = pd.concat([X, X_enc], axis = 1)
 X = pd.get_dummies(X, columns = categorical, dtype = np.int64, drop_first = True )
import itertools<concatenate>
mi = mutual_info_classif(X[features], y)
 mi_idx = mi.argsort()
for i in mi_idx:
print('{} has MI score: {}'.format(X[features].columns[i], mi[i]))
l = [all_combinations, all_combinations, all_combinations, all_combinations, all_combinations]
all_l = list(itertools.product(*l))<set_options>
X_train, X_val, y_train, y_val = train_test_split(X, y )
gc.collect()<define_variables>
features = ['Title_2.0', 'Sex_1.0', 'Fare']
filtered_combis = [l for l in all_l if l[0] + l[1] + l[2] + l[3] + l[4] > 0.95 and l[0] + l[1] + l[2] + l[3] + l[4] < 1.03]<compute_test_metric>
clfs = {
'tree' : DecisionTreeClassifier() ,
'rf' : RandomForestClassifier() ,
'extra' : ExtraTreesClassifier() ,
'grad' : GradientBoostingClassifier() ,
}
best_combi = []
for i, combi in enumerate(filtered_combis):
score1 = combi[0]
score2 = combi[1]
score3 = combi[2]
score4 = combi[3]
score5 = combi[4]
v = score1 * leak_df['pred1'].values + score2 * leak_df['pred3'].values + score3 * leak_df['pred2'].values + score4 * leak_df['pred4'] + score5 * leak_df['pred5']
vl1p = np.log1p(v)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
if best_combi:
prev_score = best_combi[0][1]
if curr_score < prev_score:
best_combi[:] = []
best_combi += [(i, curr_score)]
else:
best_combi += [(i, curr_score)]
score = best_combi[0][1]
print(score )<load_from_csv>
clf_result = []
for model_name, model in clfs.items() :
model.fit(X_train[features], y_train)
 preds = model.predict(X_test[features])
 cv_score = cross_val_score(model, X_train[features], y_train, scoring = 'accuracy', cv = 5)
 clf_result.append({
'model' : model_name,
'mean acc' : cv_score.mean() ,
'std acc' : cv_score.std()
})
 clf_result = pd.DataFrame(clf_result )
<feature_engineering>
params = {
'n_estimators' : [50, 100, 150, 200],
'learning_rate' : [0.01, 0.03, 0.05, 0.07, 0.1, 0.3],
'subsample' : [0.4, 0.45, 0.5, 0.55, 0.7],
'max_features' : ['auto', 'sqrt'],
'n_iter_no_change' : [0, 5, 10],
'max_depth' : [2, 3, 4],
}
base_clf = GradientBoostingClassifier(criterion = 'mse', validation_fraction = 0.25, random_state = 0)
 grid_search = RandomizedSearchCV(base_clf, params, scoring = 'accuracy', random_state = 0, n_iter = 40)
 grid_search.fit(X_train[features], y_train)
 print('40 random searches complete' )
submission_form = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
final_combi = filtered_combis[best_combi[0][0]]
w1 = final_combi[0]
w2 = final_combi[1]
w3 = final_combi[2]
w4 = final_combi[3]
w5 = final_combi[4]
print("Best weight is w1(kfold):{}, w2(halfhalf):{}, w3(highway):{}, w4(gep3):{}, w5(simple):{}".format(w1, w2, w3, w4, w5))
submission_form['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred3 + w3 * test_df.pred2 + w4 * test_df.pred4 + w5 * test_df.pred5
submission_form.loc[submission_form.meter_reading < 0, 'meter_reading'] = 0<set_options>
grid_search_result = pd.DataFrame(grid_search.cv_results_)
 grid_search_result.loc[grid_search_result['rank_test_score'] < 5].sort_values('rank_test_score' )
gc.collect()<drop_column>
clf_1 = GradientBoostingClassifier(n_estimators = 200,
subsample = 0.45,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 10,
learning_rate = 0.03,
validation_fraction = 0.25,
max_depth = 3,
random_state = 0)
 clf_2 = GradientBoostingClassifier(n_estimators = 50,
subsample = 0.7,
max_features = 'sqrt',
criterion = 'mse',
n_iter_no_change = 10,
learning_rate = 0.03,
validation_fraction = 0.25,
max_depth = 4,
random_state = 0)
 clf_3 = GradientBoostingClassifier(n_estimators = 150,
subsample = 0.7,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 5,
learning_rate = 0.1,
validation_fraction = 0.25,
max_depth = 2,
random_state = 0 )
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
print(len(leak_df))<feature_engineering>
final_clf = GradientBoostingClassifier(n_estimators = 150,
subsample = 0.5,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 5,
learning_rate = 0.07,
validation_fraction = 0.25,
max_depth = 3,
random_state = 0)
 plot_learning_curve(final_clf )
submission_form.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<set_options>
final_clf.fit(X_train[features], y_train)
 preds = final_clf.predict(X_val[features])
 print('Number of estimators after early stopping: {}'.format(final_clf.n_estimators_))
print('Accuracy: {}'.format(accuracy_score(preds, y_val)))
 print('F1: {}'.format(f1_score(preds, y_val)) )
gc.collect()<save_to_csv>
test = pd.read_csv('.. /input/titanic/test.csv')
 X_test = test.copy()
X_test['Fare'] = np.log1p(X_test.Fare)
 get_titles(X_test )
submission_form.to_csv('submission.csv', index=False, float_format='%.4f' )<import_modules>
X_test_imp = pd.DataFrame(missing_transformer.transform(X_test))
X_test_imp.columns = numerical + categorical
X_test.drop(numerical + categorical, axis = 1, inplace = True)
 X_test = pd.concat([X_test, X_test_imp], axis = 1 )
import pandas as pd<import_modules>
X_test_enc = pd.DataFrame(ordinal_encoder.fit_transform(X_test[categorical]))
X_test_enc.columns = categorical
X_test.drop(categorical, axis = 1, inplace = True)
 X_test = pd.concat([X_test, X_test_enc], axis = 1)
 X_test = pd.get_dummies(X_test, columns = categorical, dtype = np.int64, drop_first = True )
import pandas as pd<load_from_csv>
final_clf.fit(X[features], y)
 final_preds = final_clf.predict(X_test[features])
 final_submission = pd.DataFrame({'PassengerId' : X_test.PassengerId,
'Survived' : final_preds})
 final_submission.to_csv('may12_finalpreds.csv', index = False )