Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<compute_test_metric>
MAX_ROUNDS = 400
OPTIMIZE_ROUNDS = False
LEARNING_RATE = 0.07
EARLY_STOPPING_ROUNDS = 50
real_score = 0.55*0.6878 + 0.45*0.8175
print('Estimated Score: {0:1.3f}'.format(estimated_score))
print('Real Score: {0:1.3f}'.format(real_score))
print('Accuracy: {0:1.3f}'.format(real_score/estimated_score))<sort_values>
import numpy as np
import pandas as pd
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from numba import jit
import time
import gc
extract_from = sorted_data.drop(['surface','test','new_group_id', 'orig_id']+orientations, axis = 1)
 extract_from.head()<init_hyperparams>
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
params = {'abs_energy':None,
'absolute_sum_of_changes':None,
'agg_autocorrelation':[{'f_agg':'var','maxlag':32}],
'change_quantiles':[{'ql':0.25,'qh':0.75,'isabs':True, 'f_agg':'mean'},
{'ql':0.25,'qh':0.75,'isabs':True, 'f_agg':'std'}],
'cid_ce':[{'normalize':True},{'normalize':False}],
'fft_aggregated':[{'aggtype': 'centroid'},
{'aggtype': 'variance'},
{'aggtype': 'skew'},
{'aggtype': 'kurtosis'}],
'c3': [{'lag': 1}, {'lag': 2}, {'lag': 3}],
'standard_deviation': None,
'variance': None,
'skewness': None,
'kurtosis': None,
'maximum': None,
'minimum': None,
'sample_entropy':None,
'mean_abs_change':None,
'sum_values':None,
'quantile': [{'q': 0.1},
{'q': 0.2},
{'q': 0.3},
{'q': 0.4},
{'q': 0.6},
{'q': 0.7},
{'q': 0.8},
{'q': 0.9}],
'large_standard_deviation': [{'r': 0.25},{'r':0.35}],
'fft_coefficient': [{'coeff': 0, 'attr': 'real'},
{'coeff': 1, 'attr': 'real'},
{'coeff': 2, 'attr': 'real'},
{'coeff': 3, 'attr': 'real'},
{'coeff': 4, 'attr': 'real'},
{'coeff': 5, 'attr': 'real'},
{'coeff': 6, 'attr': 'real'},
{'coeff': 7, 'attr': 'real'},
{'coeff': 8, 'attr': 'real'},
{'coeff': 9, 'attr': 'real'},
{'coeff': 10, 'attr': 'real'},
{'coeff': 11, 'attr': 'real'},
{'coeff': 12, 'attr': 'real'},
{'coeff': 13, 'attr': 'real'},
{'coeff': 14, 'attr': 'real'},
{'coeff': 15, 'attr': 'real'},
{'coeff': 16, 'attr': 'real'},
{'coeff': 17, 'attr': 'real'},
{'coeff': 18, 'attr': 'real'},
{'coeff': 19, 'attr': 'real'},
{'coeff': 20, 'attr': 'real'},
{'coeff': 21, 'attr': 'real'},
{'coeff': 22, 'attr': 'real'},
{'coeff': 23, 'attr': 'real'},
{'coeff': 24, 'attr': 'real'},
{'coeff': 25, 'attr': 'real'},
{'coeff': 26, 'attr': 'real'},
{'coeff': 27, 'attr': 'real'},
{'coeff': 28, 'attr': 'real'},
{'coeff': 29, 'attr': 'real'},
{'coeff': 30, 'attr': 'real'},
{'coeff': 31, 'attr': 'real'},
{'coeff': 32, 'attr': 'real'},
{'coeff': 33, 'attr': 'real'},
{'coeff': 34, 'attr': 'real'},
{'coeff': 35, 'attr': 'real'},
{'coeff': 36, 'attr': 'real'},
{'coeff': 37, 'attr': 'real'},
{'coeff': 38, 'attr': 'real'},
{'coeff': 39, 'attr': 'real'},
{'coeff': 40, 'attr': 'real'},
{'coeff': 41, 'attr': 'real'},
{'coeff': 42, 'attr': 'real'},
{'coeff': 43, 'attr': 'real'},
{'coeff': 44, 'attr': 'real'},
{'coeff': 45, 'attr': 'real'},
{'coeff': 46, 'attr': 'real'},
{'coeff': 47, 'attr': 'real'},
{'coeff': 48, 'attr': 'real'},
{'coeff': 49, 'attr': 'real'},
{'coeff': 50, 'attr': 'real'},
{'coeff': 51, 'attr': 'real'},
{'coeff': 52, 'attr': 'real'},
{'coeff': 53, 'attr': 'real'},
{'coeff': 54, 'attr': 'real'},
{'coeff': 55, 'attr': 'real'},
{'coeff': 56, 'attr': 'real'},
{'coeff': 57, 'attr': 'real'},
{'coeff': 58, 'attr': 'real'},
{'coeff': 59, 'attr': 'real'},
{'coeff': 60, 'attr': 'real'},
{'coeff': 61, 'attr': 'real'},
{'coeff': 62, 'attr': 'real'},
{'coeff': 63, 'attr': 'real'},
{'coeff': 64, 'attr': 'real'}],
}<drop_column>
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
val_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_val_series = pd.merge(
 val_series.to_frame(val_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=val_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_val_series.index = val_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_val_series, noise_level), add_noise(ft_tst_series, noise_level)
extracted_features = tsfresh.extract_features(extract_from,
column_id='series_id',
column_sort='measurement_number',
n_jobs = 2,
default_fc_parameters = params )<create_dataframe>
train_df = pd.read_csv('.. /input/train.csv', na_values="-1")
 test_df = pd.read_csv('.. /input/test.csv', na_values="-1" )
efc = extracted_features.copy()
tsfresh.utilities.dataframe_functions.impute(efc)
 y = all_stitched.groupby('series_id' ).max() ['surface']
matched_y = y[y!='no_surface']
matched_ef = efc.iloc[matched_y.index]<feature_engineering>
train_features = [
"ps_car_13",
	"ps_reg_03",
	"ps_ind_05_cat",
	"ps_ind_03",
	"ps_ind_15",
	"ps_reg_02",
	"ps_car_14",
	"ps_car_12",
	"ps_car_01_cat",
	"ps_car_07_cat",
	"ps_ind_17_bin",
	"ps_car_03_cat",
	"ps_reg_01",
	"ps_car_15",
	"ps_ind_01",
	"ps_ind_16_bin",
	"ps_ind_07_bin",
	"ps_car_06_cat",
	"ps_car_04_cat",
	"ps_ind_06_bin",
	"ps_car_09_cat",
	"ps_car_02_cat",
	"ps_ind_02_cat",
	"ps_car_11",
	"ps_car_05_cat",
	"ps_calc_09",
	"ps_calc_05",
	"ps_ind_08_bin",
	"ps_car_08_cat",
	"ps_ind_09_bin",
	"ps_ind_04_cat",
	"ps_ind_18_bin",
	"ps_ind_12_bin",
	"ps_ind_14",
]
combs = [ 
('ps_reg_01', 'ps_car_02_cat'), 
('ps_reg_01', 'ps_car_04_cat'),
]
features_filtered = tsfresh.select_features(matched_ef, matched_y )<groupby>
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
start = time.time()
for n_c,(f1, f2)in enumerate(combs):
name1 = f1 + "_plus_" + f2
print('current feature %60s %4d in %5.1f'
%(name1, n_c + 1,(time.time() - start)/ 60), end='')
 print('\r' * 75, end='')
 train_df[name1] = train_df[f1].apply(lambda x: str(x)) + "_" + train_df[f2].apply(lambda x: str(x))
test_df[name1] = test_df[f1].apply(lambda x: str(x)) + "_" + test_df[f2].apply(lambda x: str(x))
lbl = LabelEncoder()
lbl.fit(list(train_df[name1].values)+ list(test_df[name1].values))
train_df[name1] = lbl.transform(list(train_df[name1].values))
test_df[name1] = lbl.transform(list(test_df[name1].values))
train_features.append(name1)
 X = train_df[train_features]
test_df = test_df[train_features]
f_cats = [f for f in X.columns if "_cat" in f]
metadata = all_stitched.groupby('series_id' ).max() [['surface','test','new_group_id', 'orig_id']]<merge>
y_valid_pred = 0*y
y_test_pred = 0
filtered_extracted = metadata.join(extracted_features[feature_columns] )<filter>
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 np.random.seed(0 )
train_filtered = filtered_extracted[filtered_extracted.test == 0]
test_filtered = filtered_extracted[filtered_extracted.test == 1]<split>
model = XGBClassifier(
 n_estimators=MAX_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
scale_pos_weight=1.6,
gamma=10,
reg_alpha=8,
reg_lambda=1.3,
)
X_filtered = train_filtered.drop(labels = ['surface','test','new_group_id', 'orig_id'],
axis = 1)
 y_filtered = train_filtered.surface
X_f_train, X_f_test, y_f_train, y_f_test = train_test_split(X_filtered, y_filtered, test_size=0.25, random_state=42, stratify = y_filtered )<normalization>
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index]
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
X_test = test_df.copy()
print("
Fold ", i)
 for f in f_cats:
X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode(
 trn_series=X_train[f],
val_series=X_valid[f],
tst_series=X_test[f],
target=y_train,
min_samples_leaf=200,
smoothing=10,
noise_level=0 
)
 if OPTIMIZE_ROUNDS:
eval_set=[(X_valid,y_valid)]
fit_model = model.fit(X_train, y_train,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose=False 
)
 print(" Best N trees = ", model.best_ntree_limit)
 print(" Best gini = ", model.best_score)
 else:
fit_model = model.fit(X_train, y_train)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] = pred
y_test_pred += fit_model.predict_proba(X_test)[:,1]
del X_test, X_train, X_valid, y_train
y_test_pred /= K
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )
scaler_f = StandardScaler()
X_f_train_scaled = scaler_f.fit_transform(X_f_train)
 X_f_test_scaled = scaler_f.transform(X_f_test )<train_model>
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('xgb_valid.csv', float_format='%.6f', index=False )
<compute_train_metric><EOS> 
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('xgb_submit.csv', float_format='%.6f', index=False )