Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<define_variables>
pd.set_option('display.max_columns', 100 )
seed = 1
np.random.seed(seed )<define_variables>
trainset = pd.read_csv('.. /input/train.csv')
 testset = pd.read_csv('.. /input/test.csv' )
ROOT = Path.cwd().parent/'input'
SAMPLE = ROOT/'sample_submission.csv'
TRAIN = ROOT/'X_train.csv'
TARGET = ROOT/'y_train.csv'
TEST = ROOT/'X_test.csv'
ID_COLS = ['series_id', 'measurement_number']
x_cols = {
'series_id': np.uint32,
'measurement_number': np.uint32,
'orientation_X': np.float32,
'orientation_Y': np.float32,
'orientation_Z': np.float32,
'orientation_W': np.float32,
'angular_velocity_X': np.float32,
'angular_velocity_Y': np.float32,
'angular_velocity_Z': np.float32,
'linear_acceleration_X': np.float32,
'linear_acceleration_Y': np.float32,
'linear_acceleration_Z': np.float32
}
y_cols = {
'series_id': np.uint32,
'group_id': np.uint32,
'surface': str
}<load_from_csv>
print("Train dataset(rows, cols):",trainset.shape, "
Test dataset(rows, cols):",testset.shape )
x_trn = pd.read_csv(TRAIN, usecols=x_cols.keys() , dtype=x_cols)
 x_tst = pd.read_csv(TEST, usecols=x_cols.keys() , dtype=x_cols)
 y_trn = pd.read_csv(TARGET, usecols=y_cols.keys() , dtype=y_cols )<categorify>
data = []
for feature in trainset.columns:
if feature == 'target':
use = 'target'
elif feature == 'id':
use = 'id'
else:
use = 'input'
if 'bin' in feature or feature == 'target':
type = 'binary'
elif 'cat' in feature or feature == 'id':
type = 'categorical'
elif trainset[feature].dtype == float or isinstance(trainset[feature].dtype, float):
type = 'real'
elif trainset[feature].dtype == int:
type = 'integer'
preserve = True
if feature == 'id':
preserve = False
dtype = trainset[feature].dtype
category = 'none'
if 'ind' in feature:
category = 'individual'
elif 'reg' in feature:
category = 'registration'
elif 'car' in feature:
category = 'car'
elif 'calc' in feature:
category = 'calculated'
feature_dictionary = {
'varname': feature,
'use': use,
'type': type,
'preserve': preserve,
'dtype': dtype,
'category' : category
}
data.append(feature_dictionary)
 metadata = pd.DataFrame(data, columns=['varname', 'use', 'type', 'preserve', 'dtype', 'category'])
 metadata.set_index('varname', inplace=True)
 metadata
def part(f, **params):

 def wrapper(x): return f(x, **params)
 wrapper.__name__ = f.__name__
return wrapper<import_modules>
metadata[(metadata.type == 'categorical')&(metadata.preserve)].index
class SliceFeatures:

 def __init__(self, mode='first', n=5):
if mode not in {'first', 'middle', 'last'}:
raise ValueError('unexpected mode')
 self.mode = mode
self.n = n
def __call__(self, data):
if self.mode == 'first':
start, end = 0, self.n
elif self.mode == 'last':
start, end = -self.n, len(data)
 elif self.mode == 'middle':
mid = len(data)// 2
div, mod = divmod(self.n, 2)
 start, end = mid-div, mid+div+mod
cols = data.columns
vec = data.iloc[start:end].values.T.ravel()
new_cols = [
f'{col}_{self.mode}{i}'
for i in range(self.n)for col in cols]
return dict(zip(new_cols, vec))<create_dataframe>
pd.DataFrame({'count' : metadata.groupby(['category'])['category'].size() } ).reset_index()
def generate_features(data, features, ignore=None):

 with Parallel(n_jobs=cpu_count())as parallel:
extracted = parallel(delayed(generate_features_for_group )(
 group=group.drop(columns=ignore or []),
features=features 
)for _, group in tqdm(data.groupby('series_id')))
 return pd.DataFrame(extracted )<categorify>
pd.DataFrame({'count' : metadata.groupby(['use', 'type'])['use'].size() } ).reset_index()
def generate_features_for_group(group, features):

 return dict(ChainMap(*[feat(group)for feat in features]))<categorify>
vars_with_missing = []
for feature in trainset.columns:
missings = trainset[trainset[feature] == -1][feature].count()
if missings > 0:
vars_with_missing.append(feature)
 missings_perc = missings/trainset.shape[0]
print('Variable {} has {} records({:.2%})with missing values'.format(feature, missings, missings_perc))
print('In total, there are {} variables with missing values'.format(len(vars_with_missing)) )
funcs =(
 mean, median, standard_deviation, variance,
skewness, kurtosis, maximum, minimum,
mean_change, mean_abs_change, count_above_mean, count_below_mean,
mean_second_derivative_central, sum_of_reoccurring_data_points,
abs_energy, sum_values, sample_entropy,
longest_strike_above_mean, longest_strike_below_mean,
first_location_of_minimum, first_location_of_maximum,
*[part(large_standard_deviation, r=r*0.05)for r in range(1, 20)],
*[part(autocorrelation, lag=lag)for lag in range(1, 25)],
*[part(number_peaks, n=n)for n in(1, 2, 3, 5, 7, 10, 25, 50)],
*[part(c3, lag=lag)for lag in range(1, 5)],
*[part(quantile, q=q)for q in (.1,.2,.3,.4,.5,.6,.7,.8,.9)],
part(partial_autocorrelation, param=[
{'lag': lag} for lag in range(25)
 ]),
part(agg_autocorrelation, param=[
{'f_agg': s, 'maxlag': 40} for s in('mean', 'median', 'var')
 ]),
part(linear_trend, param=[
{'attr': a} for a in 
('pvalue', 'rvalue', 'intercept', 'slope', 'stderr')
 ])
 )<define_variables>
col_to_drop = trainset.columns[trainset.columns.str.startswith('ps_calc_')]
trainset = trainset.drop(col_to_drop, axis=1)
 testset = testset.drop(col_to_drop, axis=1 )
features = [
StatsFeatures(funcs),
SliceFeatures('first'),
SliceFeatures('middle'),
SliceFeatures('last')
 ]<define_variables>
vars_to_drop = ['ps_car_03_cat', 'ps_car_05_cat']
trainset.drop(vars_to_drop, inplace=True, axis=1)
 testset.drop(vars_to_drop, inplace=True, axis=1)
 metadata.loc[(vars_to_drop),'keep'] = False
ignore = ['series_id', 'measurement_number']<feature_engineering>
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_tst_series, noise_level )
print('Feature extraction on train dataset')
 x_trn = generate_features(x_trn, features, ignore=ignore )<feature_engineering>
train_encoded, test_encoded = target_encode(trainset["ps_car_11_cat"],
testset["ps_car_11_cat"],
target=trainset.target,
min_samples_leaf=100,
smoothing=10,
noise_level=0.01)
 trainset['ps_car_11_cat_te'] = train_encoded
trainset.drop('ps_car_11_cat', axis=1, inplace=True)
 metadata.loc['ps_car_11_cat','keep'] = False
testset['ps_car_11_cat_te'] = test_encoded
testset.drop('ps_car_11_cat', axis=1, inplace=True )
print('Feature extraction on test dataset')
 x_tst = generate_features(x_tst, features, ignore=ignore )<categorify>
desired_apriori=0.10
idx_0 = trainset[trainset.target == 0].index
idx_1 = trainset[trainset.target == 1].index
nb_0 = len(trainset.loc[idx_0])
 nb_1 = len(trainset.loc[idx_1])
 undersampling_rate =(( 1-desired_apriori)*nb_1)/(nb_0*desired_apriori)
 undersampled_nb_0 = int(undersampling_rate*nb_0)
 print('Rate to undersample records with target=0: {}'.format(undersampling_rate))
print('Number of records with target=0 after undersampling: {}'.format(undersampled_nb_0))
undersampled_idx = shuffle(idx_0, random_state=314, n_samples=undersampled_nb_0)
 idx_list = list(undersampled_idx)+ list(idx_1)
 trainset = trainset.loc[idx_list].reset_index(drop=True )
enc = LabelEncoder()
y_trn = pd.Series(enc.fit_transform(y_trn['surface']))<compute_test_metric>
trainset = trainset.replace(-1, np.nan)
 testset = testset.replace(-1, np.nan )
def accuracy(y_true, y_pred):
n = len(y_true)
 y_hat = y_pred.reshape(9, n ).argmax(axis=0)
 value =(y_true == y_hat ).mean()
return 'accuracy', value, True<choose_model_class>
cat_features = [a for a in trainset.columns if a.endswith('cat')]
for column in cat_features:
temp = pd.get_dummies(pd.Series(trainset[column]))
trainset = pd.concat([trainset,temp],axis=1)
 trainset = trainset.drop([column],axis=1)
 for column in cat_features:
temp = pd.get_dummies(pd.Series(testset[column]))
testset = pd.concat([testset,temp],axis=1)
 testset = testset.drop([column],axis=1 )
model = lgb.LGBMClassifier(n_estimators=3000, learning_rate=0.005,
colsample_bytree=0.4, objective='multiclass',
num_leaves=500, num_class=9 )<split>
id_test = testset['id'].values
target_train = trainset['target'].values
trainset = trainset.drop(['target','id'], axis = 1)
 testset = testset.drop(['id'], axis = 1 )
x_train, x_valid, y_train, y_valid = train_test_split(x_trn, y_trn, test_size=0.1, random_state=seed )<train_model>
print("Train dataset(rows, cols):",trainset.values.shape, "
Test dataset(rows, cols):",testset.values.shape )
model.fit(x_train, y_train,
eval_set=[(x_valid, y_valid)],
eval_metric=accuracy,
early_stopping_rounds=300,
verbose=150 )<predict_on_test>
class Ensemble(object):
def __init__(self, n_splits, stacker, base_models):
self.n_splits = n_splits
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=314 ).split(X, y))
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
print("Base model %d: fit %s model | fold %d" %(i+1, str(clf ).split('(')[0], j+1))
clf.fit(X_train, y_train)
 cross_score = cross_val_score(clf, X_train, y_train, cv=3, scoring='roc_auc')
 print("cross_score [roc-auc]: %.5f [gini]: %.5f" %(cross_score.mean() , 2*cross_score.mean() -1))
y_pred = clf.predict_proba(X_holdout)[:,1]
S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict_proba(T)[:,1]
S_test[:, i] = S_test_i.mean(axis=1)
 results = cross_val_score(self.stacker, S_train, y, cv=3, scoring='roc_auc')
 print("Stacker score [gini]: %.5f" %(2 * results.mean() - 1))
self.stacker.fit(S_train, y)
 res = self.stacker.predict_proba(S_test)[:,1]
return res
test = enc.inverse_transform(model.predict(x_tst))<save_to_csv>
lgb_params1 = {}
lgb_params1['learning_rate'] = 0.02
lgb_params1['n_estimators'] = 650
lgb_params1['max_bin'] = 10
lgb_params1['subsample'] = 0.8
lgb_params1['subsample_freq'] = 10
lgb_params1['colsample_bytree'] = 0.8
lgb_params1['min_child_samples'] = 500
lgb_params1['seed'] = 314
lgb_params1['num_threads'] = 4
lgb_params2 = {}
lgb_params2['n_estimators'] = 1090
lgb_params2['learning_rate'] = 0.02
lgb_params2['colsample_bytree'] = 0.3
lgb_params2['subsample'] = 0.7
lgb_params2['subsample_freq'] = 2
lgb_params2['num_leaves'] = 16
lgb_params2['seed'] = 314
lgb_params2['num_threads'] = 4
lgb_params3 = {}
lgb_params3['n_estimators'] = 1100
lgb_params3['max_depth'] = 4
lgb_params3['learning_rate'] = 0.02
lgb_params3['seed'] = 314
lgb_params3['num_threads'] = 4
xgb_params = {}
xgb_params['objective'] = 'binary:logistic'
xgb_params['learning_rate'] = 0.04
xgb_params['n_estimators'] = 490
xgb_params['max_depth'] = 4
xgb_params['subsample'] = 0.9
xgb_params['colsample_bytree'] = 0.9
xgb_params['min_child_weight'] = 10
xgb_params['num_threads'] = 4
submit = pd.read_csv(SAMPLE)
 submit['surface'] = test
submit.to_csv('submit.csv', index=None )<import_modules>
lgb_model1 = LGBMClassifier(**lgb_params1)
 lgb_model2 = LGBMClassifier(**lgb_params2)
 lgb_model3 = LGBMClassifier(**lgb_params3)
 xgb_model = XGBClassifier(**xgb_params)
 log_model = LogisticRegression()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import math
import sklearn as sk
from sklearn import ensemble
from sklearn import model_selection
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from imblearn import under_sampling, over_sampling
from imblearn.over_sampling import SMOTE<load_from_csv>
stack = Ensemble(n_splits=3,
stacker = log_model,
base_models =(lgb_model1, lgb_model2, lgb_model3, xgb_model))
x_test = pd.read_csv('.. /input/X_test.csv')
 x_train = pd.read_csv('.. /input/X_train.csv')
 y_train = pd.read_csv('.. /input/y_train.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv')
 print('Train X: {}
Train Y: {}
Test X: {}
Submission: {}'.format(x_train.shape,y_train.shape,x_test.shape,submission.shape))<normalization>
y_prediction = stack.fit_predict(trainset, target_train, testset )
<create_dataframe><EOS> 
submission = pd.DataFrame()
submission['id'] = id_test
submission['target'] = y_prediction
submission.to_csv('stacked.csv', index=False )