Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<train_on_grid>
MAX_ROUNDS = 1200
OPTIMIZE_ROUNDS = False
LEARNING_RATE = 0.024
C_range = np.logspace(-4, 0, 4)
 param_grid = dict(C=C_range)
 SVM = GridSearchCV(SVC(kernel = 'linear'),
param_grid=param_grid,
n_jobs=-1,
pre_dispatch=2,
cv = 3)
 SVM.fit(X_f_train_scaled, y_f_train)
 y_test_svc = SVM.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_svc))<find_best_model_class>
import numpy as np
import pandas as pd
from catboost import CatBoostClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from numba import jit
from sklearn import *
import lightgbm as lgb
from multiprocessing import *
KNN = KNeighborsClassifier(10)
 KNN.fit(X_f_train_scaled,y_f_train)
 y_test_knn = KNN.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_knn))<find_best_model_class>
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
DT = DecisionTreeClassifier()
DT.fit(X_f_train_scaled,y_f_train)
 y_test_dt = DT.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_dt))<define_search_space>
def transform_df(df):
df = pd.DataFrame(df)
 dcol = [c for c in df.columns if c not in ['id','target']]
df['ps_car_13_x_ps_reg_03'] = df['ps_car_13'] * df['ps_reg_03']
df['negative_one_vals'] = np.sum(( df[dcol]==-1 ).values, axis=1)
 for c in dcol:
if '_bin' not in c:
df[c+str('_median_range')] =(df[c].values > d_median[c] ).astype(np.int)
 df[c+str('_mean_range')] =(df[c].values > d_mean[c] ).astype(np.int)
 for c in one_hot:
if len(one_hot[c])>2 and len(one_hot[c])< 7:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val ).astype(np.int)
 return df
def multi_transform(df):
p = Pool(cpu_count())
 df = p.map(transform_df, np.array_split(df, cpu_count()))
df = pd.concat(df, axis=0, ignore_index=True ).reset_index(drop=True)
 p.close() ; p.join()
return df
def gini_lgb(preds, dtrain):
y = list(dtrain.get_label())
 score = eval_gini(y, preds)/ eval_gini(y, y)
 return 'gini', score, True
param_grid = {
'max_depth': [3,4,5,6],
'min_child_weight':np.linspace(0.8,1.2,4),
'gamma': np.linspace(0,0.2,4),
}<choose_model_class>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
XGB = xgb.sklearn.XGBClassifier(learning_rate = 0.025,
objective = 'multi:softmax',
n_estimators = 150,
max_depth = 5,
min_child_weight = 1.2,
subsample=0.8,
colsample_bytree = 0.8,
gamma = 0.066,
n_jobs = 4,
nthreads = 1,
silent = True,
seed = 42)
 XGB.fit(X_f_train_scaled,y_f_train)
 y_test_XGB = XGB.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_svc))<predict_on_test>
col = [c for c in train_df.columns if c not in ['id','target']]
col = [c for c in col if not c.startswith('ps_calc_')]
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
X = train_df[col]
y_valid_pred = 0*y
X_test = test_df.drop(['id'], axis=1)
 y_test_pred = 0
def get_predictions(filtered_data, classifiers, scaler):
warnings.filterwarnings("ignore")
 df = filtered_data[['surface','test','new_group_id', 'orig_id']]
X = filtered_data.drop(labels = ['surface','test','new_group_id','orig_id'],
axis = 1 ).values
X_scaled = scaler.transform(X)
 for name, classifier_dict in classifiers.items() :
clf = classifier_dict['classifier']
if classifier_dict['scale']:
prediction = clf.predict(X_scaled)
 else:
prediction = clf.predict(X)
 df[name] = prediction
warnings.resetwarnings()
return df<normalization>
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True )
classifiers = {'RFC':{'classifier':RFC,'scale': False},
'NB':{'classifier':NB,'scale':True},
'SVM':{'classifier':SVM,'scale':True},
'KNN':{'classifier':KNN,'scale':True},
'DT':{'classifier':DT,'scale':True},
'XGB':{'classifier':XGB,'scale':True}}<predict_on_test>
params = {
'learning_rate': LEARNING_RATE,
'max_depth': 4,
'lambda_l1': 16.7,
'boosting': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'feature_fraction':.7,
'is_training_metric': False,
'seed': 99
}
multi_predictions = get_predictions(filtered_data=train_filtered,
classifiers=classifiers,
scaler=scaler_f )<groupby>
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index].copy()
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
test = test_df.copy() [col]
print("
Fold ", i)
 one_hot = {c: list(X_train[c].unique())for c in X_train.columns}
X_train = X_train.replace(-1, np.NaN)
 d_median = X_train.median(axis=0)
 d_mean = X_train.mean(axis=0)
 X_train = X_train.fillna(-1)
 X_train = multi_transform(X_train)
 X_valid = multi_transform(X_valid)
 test = multi_transform(test)
 if OPTIMIZE_ROUNDS:
fit_model = lgb.train(
 params,
lgb.Dataset(X_train, label=y_train),
MAX_ROUNDS,
lgb.Dataset(X_valid, label=y_valid),
verbose_eval=50,
feval=gini_lgb,
early_stopping_rounds=200 
)
 print(" Best iteration = ", fit_model.best_iteration)
 pred = fit_model.predict(X_valid, num_iteration=fit_model.best_iteration)
 test_pred = fit_model.predict(test[col], num_iteration=fit_model.best_iteration)
 else:
fit_model = lgb.train(
 params,
lgb.Dataset(X_train, label=y_train),
MAX_ROUNDS,
verbose_eval=50 
)
 pred = fit_model.predict(X_valid)
 test_pred = fit_model.predict(test)
 print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] =(np.exp(pred)- 1.0 ).clip(0,1)
 y_test_pred +=(np.exp(test_pred)- 1.0 ).clip(0,1)
 y_test_pred /= K
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )
def classifier_ensemble(predictions, classifier_weights, groupby = 'new_group_id'):
warnings.filterwarnings("ignore")
 df = predictions[['test','new_group_id','orig_id']]
df['surface'] = 'None'
g = predictions.groupby(groupby)
 for name, group in g:
index = group.index
weighted_value_counts = []
for classifier,weight in classifier_weights.items() :
value_counts = group[classifier].value_counts()
weighted_value_counts.append(value_counts*weight)
 all_counts = pd.DataFrame(weighted_value_counts ).fillna(value=0)
 surface = all_counts.sum().sort_values(ascending=False ).index[0]
df.loc[index,'surface'] = surface
warnings.resetwarnings()
return df<choose_model_class>
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('lgb_valid.csv', float_format='%.6f', index=False )
<compute_test_metric><EOS> 
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('lgb_submit.csv', float_format='%.6f', index=False )