Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<feature_engineering>
import os
import pprint as pp
import math as ma
import numpy as np
import pandas as pd
from plotnine import *
from plotnine.data import *
from sklearn import base
from sklearn import tree
from sklearn import naive_bayes as nb
from sklearn import svm as svm
from sklearn import neighbors as nei
from sklearn import preprocessing as pre
from sklearn import feature_extraction as fe
from sklearn import pipeline as pi
from sklearn import model_selection as ms
from sklearn import ensemble as ens
from sklearn.metrics import make_scorer
import xgboost as xgb
import bayes_opt as bo
import category_encoders as ce
def find_matches(current_signal, df, in_front = True, threshold = 0.01):
orientations = ['orientation_X','orientation_Y','orientation_Z','orientation_W']
selection = ['series_id'] + orientations
if in_front:
end = current_signal.iloc[127][selection]
front = df.loc[df.measurement_number ==0][selection]
else:
front = current_signal.iloc[0][selection]
end = df.loc[df.measurement_number ==127][selection]
compare =(front-end ).abs()
if in_front:
compare.series_id = front.series_id
else:
compare.series_id = end.series_id
compare['sums'] = compare[orientations].sum(axis=1)
 compare['in_front'] = in_front
compare.drop(labels = orientations, axis = 1, inplace = True)
 compare.sort_values(by = 'sums', inplace = True)
 if any(compare.sums < threshold):
return compare.loc[compare.sums < threshold], True
else:
return 0,False<concatenate>
verbose = 10
seed = round(ma.pi * 10**4 )
def continuity(current_signal, compare_signal, in_front = True, merge_threshold = 4, slope_threshold = 6):
orientations = ['orientation_X','orientation_Y','orientation_Z','orientation_W']
select = ['series_id'] + orientations
if in_front:
end = current_signal[select]
front = compare_signal[select]
else:
front = current_signal[select]
end = compare_signal[select]
stitched = pd.concat([end,front])
 diff = stitched.diff()
mean = diff[orientations].iloc[120:127].mean()
std = diff[orientations].iloc[120:127].std()
test =(diff[orientations].iloc[128]-mean)/std
if any(test.abs() > merge_threshold):
return False
slope_front = diff[orientations].iloc[120:127].mean()
std_front = diff[orientations].iloc[120:127].std()
var_front = std_front.apply(np.square)
 slope_end = diff[orientations].iloc[129:136].mean()
std_end = diff[orientations].iloc[129:136].std()
var_end = std_end.apply(np.square)
 t_value =(slope_front-slope_end)/(var_front/8+var_end/8 ).apply(np.sqrt)
 if any(t_value.abs() > slope_threshold):
return False
return True<concatenate>
def cols_types_to_df(df, cols):
return pd.DataFrame(df[cols].dtypes.values.reshape(1, -1), columns=cols, index=['dtype'] )
def recast_df(df, match_threshold = 0.01, continuity_thresholds =(4,6)) :
lookup_dict = group_series(df,
match_threshold=match_threshold,
continuity_thresholds=continuity_thresholds)
 sub_dfs = []
for key,group in lookup_dict.items() :
temp_df = stitch_series(df,group)
 temp_df['new_group_id'] = key
sub_dfs.append(temp_df)
 stitched = pd.concat(sub_dfs)
 return stitched<create_dataframe>
ID_COL = 'id'
TARGET_COL = 'target'
SPECIAL_COL_NAMES = [ID_COL, TARGET_COL]
train = pd.read_csv('.. /input/train.csv' ).drop(ID_COL, axis=1)
 headers = train.columns
X = train.drop(TARGET_COL, axis=1)
 y = train[TARGET_COL]
train.head(10 )
stitched_train = recast_df(train, match_threshold=0.05, continuity_thresholds=(6,9))<groupby>
target_counts = train[TARGET_COL].value_counts()
target_ratio = target_counts[0]/target_counts[1]
print("Proporção entre classes 0 e 1: %s" % target_ratio )
g = stitched_train.groupby('new_group_id')
 surface_count = []
sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max Surface Count:{0}'.format(max(surface_count)))
 print('Min Sample Count:{0}'.format(min(sample_count)))
 print('Max Sample Count:{0}'.format(max(sample_count)) )<count_unique_values>
def drop_cols_func(X):

 return X.drop(columns=CALC_COL_NAMES)
 drop_cols = pre.FunctionTransformer(func=drop_cols_func, validate=False)
 drop_cols.transform(X ).head()
g = stitched_train.groupby('group_id')
 sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max sample count in original groups:{0}'.format(max(sample_count)) )<feature_engineering>
one_hot = ce.OneHotEncoder(cols=CAT_COL_NAMES, drop_invariant=True, handle_unknown='impute')
 one_hot.fit_transform(X ).head()
train.drop(['row_id','group_id'], axis = 1, inplace = True)
 train['orig_id'] = train.series_id
train['test'] = 0
test.drop(['row_id'], axis = 1, inplace = True)
 test['surface'] = 'no_surface'
test['orig_id'] = test.series_id
test['test'] = 1
test.series_id += train.series_id.max() +1<concatenate>
def gini(actual, pred):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(actual, pred):
return gini(actual, pred[:,1])/ gini(actual, actual)
 gini_normalized_scorer = make_scorer(score_func=gini_normalized, greater_is_better=True, needs_proba=True)
 def cv_metric(estimator, X, y, fit_params=None):
pred = ms.cross_val_predict(estimator, X, y, cv=ms.StratifiedKFold(n_splits=3, random_state=seed), verbose=verbose, fit_params=fit_params, method='predict_proba')
 return gini_normalized(y, pred)
 def xgb_metric(pred, dtrain):
print('xgb_gini_normalized')
 return 'gini', gini_normalized(dtrain.get_labels() , pred)
 xgb_fit_params = {
'clf__eval_metric': xgb_metric
}
all_data = pd.concat([train,test], sort=False, ignore_index=True )<create_dataframe>
xgb_params = {
'objective': 'binary:logistic'
}
def pipeline() :
steps = [('drop_cols', drop_cols), 
('one_hot', one_hot), 
('clf', xgb.XGBClassifier(random_state=seed)) ]
return pi.Pipeline(steps)
 pipe = pipeline()
all_stitched = recast_df(all_data, match_threshold=0.05, continuity_thresholds=(9,9))<groupby>
X_train, X_test, y_train, y_test = ms.train_test_split(X, y, test_size=0.33, random_state=seed )
g = all_stitched.groupby('new_group_id')
 surface_count = []
sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max Surface Count:{0}'.format(max(surface_count)))
 print('Min Sample Count:{0}'.format(min(sample_count)))
 print('Max Sample Count:{0}'.format(max(sample_count)) )<groupby>
g = all_stitched.groupby('new_group_id')
 new_dfs = []
error_surfaces = []
error_group_ids = []
matched = 0
unmatched = 0
unmatched_ids = []
for name, group in g:
surfaces = list(group.surface.unique())
 if(len(surfaces)==2)&('no_surface' in surfaces):
surfaces.remove('no_surface')
 group['surface'] = surfaces[0]
new_dfs.append(group)
 matched +=1
elif(len(surfaces)==2)&('no_surface' not in surfaces):
error_surfaces.append(surfaces)
 new_dfs.append(group)
 elif(len(surfaces)>2):
error_surfaces.append(surfaces)
 error_group_ids.append(name)
 new_dfs.append(group)
 else:
new_dfs.append(group)
 if surfaces[0] == 'no_surface':
unmatched += 1
unmatched_ids.append(name)
 sorted_data = pd.concat(new_dfs)
 sorted_data.reset_index(inplace=True, drop = True )<groupby>
if os.path.isfile('cache.csv'):
best_configs = pd.read_csv('cache.csv' ).sort_values(by='target', ascending=False)
 best_configs = best_configs.head(5 ).drop(columns='target' ).to_dict(orient='list')
 pp.pprint(best_configs)
 else:
best_configs = {'colsample_bylevel': [1.0, 1.0, 1.0, 1.0, 0.6210536972328008],
'colsample_bytree': [0.1639149309112309, 0.4719120160563106, 0.2738881448393379, 0.9597000000000002, 0.4998605743150732],
'gamma': [9.311242926501441, 9.106399343640415, 9.84350195229832, 1.3079, 8.495594735754976],
'learning_rate': [0.1, 0.1, 0.1, 0.1, 0.3],
'max_delta_step': [0.0, 0.0, 0.0, 0.0, 9.937144736135796],
'max_depth': [3.280267137915809, 3.744769687598659, 3.079866163690399, 3.8448, 3.0243950675489177],
'min_child_weight': [1.0, 1.0, 1.0, 1.0, 1.0],
'n_estimators': [496.3697387347585, 496.70662978009966, 362.50656654507236, 499.9892, 350.3584899983292],
'reg_alpha': [9.79619399515934, 0.7119072322272911, 0.05797462113553141, 8.128199999999998, 4.196426179286616],
'reg_lambda': [1.0, 1.0, 1.0, 1.0, 9.729243825652572],
'scale_pos_weight': [2.091440738145689, 1.7334510787353241, 1.327362597470453, 2.6812, 1.072600468578576],
'subsample': [0.7142488578466437, 0.9885285182085388, 0.7352809336635373, 0.9199, 0.8611579899201934]}
print('Number of unmatched groups:{0}'.format(unmatched))<define_variables>
def to_xgb_params(**kwargs):
params = {}
params['clf__max_depth'] = int(kwargs['max_depth'])
 params['clf__learning_rate'] = max(kwargs['learning_rate'], 0)
 params['clf__n_estimators'] = int(kwargs['n_estimators'])
 params['clf__min_child_weight'] = int(kwargs['min_child_weight'])
 params['clf__max_delta_step'] = int(kwargs['max_delta_step'])
 params['clf__subsample'] = max(min(kwargs['subsample'], 1), 0)
 params['clf__colsample_bytree'] = max(min(kwargs['colsample_bytree'], 1), 0)
 params['clf__colsample_bylevel'] = max(min(kwargs['colsample_bylevel'], 1), 0)
 params['clf__gamma'] = max(kwargs['gamma'], 0)
 params['clf__reg_alpha'] = max(kwargs['reg_alpha'], 0)
 params['clf__scale_pos_weight'] = max(kwargs['scale_pos_weight'], 0.001)
 return params
def xgb_cv(**kwargs):
xgb = base.clone(pipe)
 xgb.set_params(**to_xgb_params(**kwargs))
return cv_metric(xgb, X_train, y_train, fit_params=xgb_fit_params)
 bo_opt = bo.BayesianOptimization(xgb_cv, {'max_depth':(3, 15),
'learning_rate':(0.1, 0.0001),
'n_estimators':(100, 500),
'min_child_weight':(1, 1),
'max_delta_step':(0, 10),
'subsample':(0.5, 1.0),
'colsample_bytree':(0.1, 1.0),
'colsample_bylevel':(0.1, 1.0),
'gamma':(0.0, 10.0),
'reg_alpha':(0.0, 10.0),
'reg_lambda':(1.0, 10.0),
'scale_pos_weight':(1.0, target_ratio),
},
random_state=seed)

best_config = {'colsample_bylevel': 0.45879276636748534,
'colsample_bytree': 0.7824778832374746,
'gamma': 5.489188804299045,
'learning_rate': 0.1,
'max_delta_step': 1.5254539296007386,
'max_depth': 3.4967941454908598,
'min_child_weight': 1.0,
'n_estimators': 461.216046669665,
'reg_alpha': 8.531590560075347,
'reg_lambda': 9.89766553793669,
'scale_pos_weight': 1.2379042177844632,
'subsample': 0.798095844721928}
opt_params = to_xgb_params(**best_config)
 opt_pipe = base.clone(pipe)
 opt_pipe.set_params(**opt_params )
def reassign_surfaces(new_surface_list, group_id, df):
error_group = df[sorted_data.new_group_id == group_id]
reassign = {}
start = 1
end = 1
segment = 0
escape = 0
while escape == 0:
temp_dict = {}
surface = error_group.surface.iloc[0]
start = error_group.first_valid_index()
if len(error_group.surface.unique())> 1:
end = error_group.loc[error_group.surface != surface].first_valid_index()
else:
end = error_group.last_valid_index()
escape = 1
error_group.drop(range(start,end),inplace = True)
 if surface == 'no_surface':
temp_dict['surface'] = new_surface_list[segment]
temp_dict['start'] = start
temp_dict['end'] = end
reassign[segment] = temp_dict
segment += 1
for key,value in reassign.items() :
df.iloc[value['start']:value['end'],1] = value['surface']
fixed_group = df[sorted_data.new_group_id == group_id]
plt.figure(figsize =(12,4))
sns.lineplot(x = 'measurement_number',
y = 'orientation_X',
data = fixed_group,
hue='surface');<concatenate>
pipe.fit(X_train, y_train, **xgb_fit_params)
 print('Configuração padrão: {}'.format(gini_normalized(y_test, pipe.predict_proba(X_test))))
opt_pipe.fit(X_train, y_train, **xgb_fit_params)
 print('Configuração otimizada: {}'.format(gini_normalized(y_test, opt_pipe.predict_proba(X_test))))
surface_2 = ['concrete','wood','wood']
reassign_surfaces(surface_2,2,sorted_data )<concatenate>
class FitParamsWrapper(base.BaseEstimator, base.ClassifierMixin):
def __init__(self, estimator, fit_params):
self.estimator = estimator
self.fit_params = fit_params
def fit(self, X, y=None):
self.estimator.fit(X, y, **self.fit_params)
 return self
def predict(self, X, y=None):
return self.estimator.predict(X)
 def predict_proba(self, X, y=None):
return self.estimator.predict_proba(X)
 def score(self, X, y=None):
return self.estimator.score(X, y )
<save_to_csv><EOS> 
opt_pipe.fit(X, y, **xgb_fit_params)
 test = pd.read_csv('.. /input/test.csv')
 X_test = test[X.columns]
targets = opt_pipe.predict_proba(X_test)
 targets = pd.DataFrame({'id': test.id, 'target': targets[:,1]})
 targets.to_csv('submission.csv', index=False )