Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<feature_engineering>
import numpy as np
import pandas as pd
from xgboost import XGBClassifier
from sklearn.model_selection import KFold,StratifiedKFold
from numba import jit
x_train = add_direction_vectors(x_train)
 x_test = add_direction_vectors(x_test)
 x_train.head()<feature_engineering>
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
preds -= preds.min()
preds / preds.max()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
val_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_val_series = pd.merge(
 val_series.to_frame(val_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=val_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_val_series.index = val_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_val_series, noise_level), add_noise(ft_tst_series, noise_level )
stats = ['norm_orientation_X', 'norm_orientation_Y', 'norm_orientation_Z',
'norm_orientation_W',
'norm_velocity_X', 'norm_velocity_Y', 'norm_velocity_Z',
'norm_acceleration_X', 'norm_acceleration_Y', 'norm_acceleration_Z',
'mod_linear_acceleration',
'ratio_velocity-acceleration',
'orientation_vector_X', 'orientation_vector_Y', 'orientation_vector_Z']
train_features = descriptive_features(train_features, x_train, stats)
 test_features = descriptive_features(test_features, x_test, stats )<train_model>
strdirectory = '.. /input/'
train = pd.read_csv(strdirectory+'train.csv')
 test = pd.read_csv(strdirectory+'test.csv')
 test.insert(1,'target',0)
 print(train.shape)
 print(test.shape)
 x = pd.concat([train,test])
 x = x.reset_index(drop=True)
 unwanted = x.columns[x.columns.str.startswith('ps_calc_')]
x.drop(unwanted,inplace=True,axis=1)
 x.loc[:,'ps_reg_03'] = pd.cut(x['ps_reg_03'], 50,labels=False)
 x.loc[:,'ps_car_12'] = pd.cut(x['ps_car_12'], 50,labels=False)
 x.loc[:,'ps_car_13'] = pd.cut(x['ps_car_13'], 50,labels=False)
 x.loc[:,'ps_car_14'] = pd.cut(x['ps_car_14'], 50,labels=False)
 x.loc[:,'ps_car_15'] = pd.cut(x['ps_car_15'], 50,labels=False)
 test = x.iloc[train.shape[0]:].copy()
train = x.iloc[:train.shape[0]].copy()
train_features = eng_features(train_features)
 test_features = eng_features(test_features)
 print("Train features: ", train_features.shape)
 print("Test features: ", test_features.shape )<drop_column>
features = train.columns[2:]
ranktestpreds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,shuffle=True,random_state=2017)
 for i,(train_index, test_index)in enumerate(kf.split(list(train.index))):
print('Fold: ',i)
 myfeatures = list(features[:])
 blindtrain = train.iloc[test_index].copy()
vistrain = train.iloc[train_index].copy()
mytest = test.copy()
for column in features:
vis, blind, tst = target_encode(trn_series=vistrain[column],
val_series=blindtrain[column],
tst_series=mytest[column],
target=vistrain.target,
min_samples_leaf=200,
smoothing=10,
noise_level=0)
 vistrain['te_' + column] = vis
blindtrain['te_' + column] = blind
mytest['te_' + column] = tst
myfeatures = myfeatures + list(['te_' + column])
 clf = XGBClassifier(n_estimators=2000,
objective="rank:pairwise",
learning_rate = 0.04,
max_depth = 5,
min_child_weight = 9,
subsample = 0.8,
colsample_bytree = 0.8,
reg_alpha = 10.4,
reg_lambda = 0.59,
seed = 2017,
nthread = 8,
silent = 1)
 eval_set=[(blindtrain[myfeatures],blindtrain.target)]
model = clf.fit(vistrain[myfeatures], vistrain.target,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=70,
verbose=False)
 print(" Best N trees = ", model.best_ntree_limit)
 print(" Best gini = ", model.best_score)
 trainpreds = model.predict_proba(blindtrain[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
print(" Best gini = ", eval_gini(blindtrain.target,trainpreds))
ranktestpreds += model.predict_proba(mytest[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
ranktestpreds /= 5
ranktestpreds -= ranktestpreds.min()
ranktestpreds /= ranktestpreds.max()
drops = []
threshold =.95
for i in range(len(top_corr)) :
if top_corr.iloc[i]["raw corr"] > threshold:
if top_corr.iloc[i]["var1"] not in drops and top_corr.iloc[i]["var2"] not in drops:
drops.append(top_corr.iloc[i]["var2"])
 train_features = train_features.drop(columns=drops)
 test_features = test_features.drop(columns=drops )<prepare_x_and_y>
xgbtestpreds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,shuffle=True,random_state=2017)
 for i,(train_index, test_index)in enumerate(kf.split(list(train.index))):
print('Fold: ',i)
 myfeatures = list(features[:])
 blindtrain = train.iloc[test_index].copy()
vistrain = train.iloc[train_index].copy()
mytest = test.copy()
for column in features:
vis, blind, tst = target_encode(trn_series=vistrain[column],
val_series=blindtrain[column],
tst_series=mytest[column],
target=vistrain.target,
min_samples_leaf=200,
smoothing=10,
noise_level=0)
 vistrain['te_' + column] = vis
blindtrain['te_' + column] = blind
mytest['te_' + column] = tst
myfeatures = myfeatures + list(['te_' + column])
 clf = XGBClassifier(n_estimators=2000,
learning_rate = 0.04,
max_depth = 5,
min_child_weight = 9,
subsample = 0.8,
colsample_bytree = 0.8,
reg_alpha = 10.4,
reg_lambda = 0.59,
seed = 2017,
nthread = 8,
silent = 1)
 eval_set=[(blindtrain[myfeatures],blindtrain.target)]
model = clf.fit(vistrain[myfeatures], vistrain.target,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=70,
verbose=False)
 print(" Best N trees = ", model.best_ntree_limit)
 print(" Best gini = ", model.best_score)
 trainpreds = model.predict_proba(blindtrain[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
print(" Best gini = ", eval_gini(blindtrain.target,trainpreds))
xgbtestpreds += model.predict_proba(mytest[myfeatures],ntree_limit=model.best_ntree_limit)[:,1]
xgbtestpreds /= 5
xgbtestpreds -= xgbtestpreds.min()
xgbtestpreds /= xgbtestpreds.max()
seed = 1920348
sm = SMOTE(sampling_strategy='not majority', random_state=seed, k_neighbors=10)
 y_train_rs = pd.DataFrame()
train_features_rs, y_train_rs['surface'] = sm.fit_resample(train_features, y_train['surface'] )<prepare_x_and_y>
rankdata = pd.DataFrame()
rankdata['xgbnormal'] = xgbtestpreds
rankdata['xgbrank'] = ranktestpreds
<compute_test_metric><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.target =(rankdata.xgbnormal.rank() +rankdata.xgbrank.rank())
 sub.target -= sub.target.min()
sub.target /= sub.target.max()
sub.to_csv('xgbsubmission.csv', index = False )