Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<predict_on_test>
USE_RGF_INSTEAD = True
MAX_XGB_ROUNDS = 400
OPTIMIZE_XGB_ROUNDS = False
XGB_LEARNING_RATE = 0.07
XGB_EARLY_STOPPING_ROUNDS = 50
submission['surface'] = final_model.predict(test_features)
 submission.head()<save_to_csv>
import numpy as np
import pandas as pd
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from rgf.sklearn import RGFClassifier
from numba import jit
import time
import gc
import subprocess
import glob
submission.to_csv('final_submission_smote.csv', index = False )<import_modules>
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
import itertools
import xgboost as xgb
import time
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import *<load_from_csv>
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
val_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_val_series = pd.merge(
 val_series.to_frame(val_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=val_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_val_series.index = val_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_val_series, noise_level), add_noise(ft_tst_series, noise_level)
X_train = pd.read_csv(".. /input/X_train.csv")
 X_test = pd.read_csv(".. /input/X_test.csv")
 y_train = pd.read_csv(".. /input/y_train.csv")
 sub = pd.read_csv(".. /input/sample_submission.csv" )<feature_engineering>
train_df = pd.read_csv('.. /input/train.csv', na_values="-1")
 test_df = pd.read_csv('.. /input/test.csv', na_values="-1" )
def fe(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df 
<categorify>
train_features = [
"ps_car_13",
	"ps_reg_03",
	"ps_ind_05_cat",
	"ps_ind_03",
	"ps_ind_15",
	"ps_reg_02",
	"ps_car_14",
	"ps_car_12",
	"ps_car_01_cat",
	"ps_car_07_cat",
	"ps_ind_17_bin",
	"ps_car_03_cat",
	"ps_reg_01",
	"ps_car_15",
	"ps_ind_01",
	"ps_ind_16_bin",
	"ps_ind_07_bin",
	"ps_car_06_cat",
	"ps_car_04_cat",
	"ps_ind_06_bin",
	"ps_car_09_cat",
	"ps_car_02_cat",
	"ps_ind_02_cat",
	"ps_car_11",
	"ps_car_05_cat",
	"ps_calc_09",
	"ps_calc_05",
	"ps_ind_08_bin",
	"ps_car_08_cat",
	"ps_ind_09_bin",
	"ps_ind_04_cat",
	"ps_ind_18_bin",
	"ps_ind_12_bin",
	"ps_ind_14",
]
combs = [ 
('ps_reg_01', 'ps_car_02_cat'), 
('ps_reg_01', 'ps_car_04_cat'),
]
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'])
<categorify>
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
start = time.time()
for n_c,(f1, f2)in enumerate(combs):
name1 = f1 + "_plus_" + f2
print('current feature %60s %4d in %5.1f'
%(name1, n_c + 1,(time.time() - start)/ 60), end='')
 print('\r' * 75, end='')
 train_df[name1] = train_df[f1].apply(lambda x: str(x)) + "_" + train_df[f2].apply(lambda x: str(x))
test_df[name1] = test_df[f1].apply(lambda x: str(x)) + "_" + test_df[f2].apply(lambda x: str(x))
lbl = LabelEncoder()
lbl.fit(list(train_df[name1].values)+ list(test_df[name1].values))
train_df[name1] = lbl.transform(list(train_df[name1].values))
test_df[name1] = lbl.transform(list(test_df[name1].values))
train_features.append(name1)
 X = train_df[train_features]
test_df = test_df[train_features]
f_cats = [f for f in X.columns if "_cat" in f]
X_train.fillna(0, inplace = True)
 X_test.fillna(0, inplace = True)
 X_train.replace(-np.inf, 0, inplace = True)
 X_train.replace(np.inf, 0, inplace = True)
 X_test.replace(-np.inf, 0, inplace = True)
 X_test.replace(np.inf, 0, inplace = True )<split>
y_valid_pred = 0*y
y_test_pred = 0
kfold = StratifiedKFold(n_splits=5)
 train_x,val_x,train_y,val_y = train_test_split(data, target, test_size = 0.10, random_state=14)
 train_x.shape,val_x.shape,train_y.shape,val_y.shape<create_dataframe>
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 np.random.seed(0 )
train_data = lightgbm.Dataset(train_x, label=train_y)
 test_data = lightgbm.Dataset(val_x, label=val_y )<train_model>
xgbmodel = XGBClassifier(
 n_estimators=MAX_XGB_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=XGB_LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
scale_pos_weight=1.6,
gamma=10,
reg_alpha=8,
reg_lambda=1.3, 
)
 rgf = RGFClassifier(
 max_leaf=1200,
algorithm="RGF",
loss="Log",
l2=0.01,
sl2=0.01,
normalize=False,
min_samples_leaf=10,
n_iter=None,
opt_interval=100,
learning_rate=.5,
calc_prob="sigmoid",
n_jobs=-1,
memory_policy="generous",
verbose=0
)
para={'boosting_type': 'gbdt',
'colsample_bytree': 0.85,
'learning_rate': 0.1,
'max_bin': 512,
'max_depth': -1,
'metric': 'multi_error',
'min_child_samples': 8,
'min_child_weight': 1,
'min_split_gain': 0.5,
'nthread': 3,
'num_class': 9,
'num_leaves': 31,
'objective': 'multiclass',
'reg_alpha': 0.8,
'reg_lambda': 1.2,
'scale_pos_weight': 1,
'subsample': 0.7,
'subsample_for_bin': 200,
'subsample_freq': 1}
model = lightgbm.train(para,
train_data,
valid_sets=test_data,
num_boost_round=5000,
early_stopping_rounds=50,
)<predict_on_test>
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index]
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
X_test = test_df.copy()
print("
Fold ", i)
 for f in f_cats:
X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode(
 trn_series=X_train[f],
val_series=X_valid[f],
tst_series=X_test[f],
target=y_train,
min_samples_leaf=200,
smoothing=10,
noise_level=0 
)
 if USE_RGF_INSTEAD:
X_train = X_train.fillna(X_train.mean())
 rgf.fit(X_train, y_train)
 elif OPTIMIZE_XGB_ROUNDS:
eval_set=[(X_valid,y_valid)]
fit_model = xgbmodel.fit(X_train, y_train,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=XGB_EARLY_STOPPING_ROUNDS,
verbose=False 
)
 print(" Best N trees = ", xgbmodel.best_ntree_limit)
 print(" Best gini = ", xgbmodel.best_score)
 else:
fit_model = xgbmodel.fit(X_train, y_train)
 if USE_RGF_INSTEAD:
pred = rgf.predict_proba(X_valid.fillna(X_train.mean())) [:,1]
else:
pred = fit_model.predict_proba(X_valid)[:,1]
print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] = pred
if USE_RGF_INSTEAD:
probs = rgf.predict_proba(X_test.fillna(X_train.mean())) [:,1]
try:
subprocess.call('rm -rf /tmp/rgf/*', shell=True)
 print("Clean up is successfull")
 print(glob.glob("/tmp/rgf/*"))
except Exception as e:
print(str(e))
else:
probs = fit_model.predict_proba(X_test)[:,1]
almost_zero = 1e-12
almost_one = 1 - almost_zero
probs[probs>almost_one] = almost_one
probs[probs<almost_zero] = almost_zero
y_test_pred += np.log(probs/(1-probs))
del X_test, X_train, X_valid, y_train
y_test_pred /= K
y_test_pred = 1 /(1 + np.exp(-y_test_pred))
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )
y_pred = model.predict(X_test )<create_dataframe>
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('kfloa_valid.csv', float_format='%.6f', index=False )
<count_values><EOS> 
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('kfloa_submit.csv', float_format='%.6f', index=False )