Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<categorify>
random_state = 6
np.random.seed(random_state)
 warnings.filterwarnings('ignore' )
train_df.fillna(0, inplace=True)
 test_df.fillna(0, inplace=True)
 train_df.replace(-np.inf, 0, inplace=True)
 train_df.replace(np.inf, 0, inplace=True)
 test_df.replace(-np.inf, 0, inplace=True)
 test_df.replace(np.inf, 0, inplace=True )<normalization>
%matplotlib inline
get_ipython().run_line_magic('matplotlib', 'inline')
 font = {
'family': 'serif',
'serif': ['Computer Modern Roman'],
'weight' : 'regular',
'size' : 18
}
plt.rc('font', **font)
 plt.rc('text', usetex=False)
 color_map = 'viridis'
sc = StandardScaler()
train_df = sc.fit_transform(train_df)
 test_df = sc.transform(test_df )<predict_on_test>
df_train = pd.read_csv('/kaggle/input/train.csv')
 df_test = pd.read_csv('/kaggle/input/test.csv' )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=111222)
 sub_preds_rf = np.zeros(( test_df.shape[0],9))
oof_preds_rf = np.zeros(( train_df.shape[0]))
score = 0
counter = 0
print('start training')
 for train_index, test_index in folds.split(train_df, target_df['surface']):
print('Fold {}'.format(counter+1))
clf = RandomForestClassifier(n_estimators=200, n_jobs=-1)
 clf.fit(train_df[train_index], target_df['surface'][train_index])
 oof_preds_rf[test_index] = clf.predict(train_df[test_index])
 sub_preds_rf += clf.predict_proba(test_df)/ folds.n_splits
score += clf.score(train_df[test_index], target_df['surface'][test_index])
 counter += 1
print('score : {}'.format(clf.score(train_df[test_index], target_df['surface'][test_index])))
 print('avg accuracy : {}'.format(score / folds.n_splits)) 
<save_to_csv>
print('Training data shape: {}'.format(df_train.shape))
print('Training data shape: {}'.format(df_test.shape))
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
submit.to_csv('submit.csv', index=False)
 print('Ready' )<define_variables>
print('Is null on train: {}'.format(df_train.isnull().any().any()))
print('Is null on test: {}'.format(df_test.isnull().any().any()))
PATH = '.. /input/career-con-2019'<load_from_csv>
col_to_drop = list(df_train.columns[df_train.columns.str.startswith('ps_calc_')])
 df_train = df_train.drop(col_to_drop, axis=1)
 df_test = df_test.drop(col_to_drop, axis=1 )
gc.enable()
train_x = pd.read_csv(PATH+'/X_train.csv')
 train_y = pd.read_csv(PATH+'/y_train.csv' )<create_dataframe>
def get_missing_features(df):
missings = pd.DataFrame([], columns=['feature', 'no_recoreds', 'percentage'])
 total_rows = df.shape[0]
index = 0
for feature in list(df):
total_nulls = df[feature].isnull().sum()
if total_nulls > 0:
missings_perc = total_nulls / total_rows
missings.loc[index] = [feature, total_nulls, missings_perc]
index += 1
missings = missings.sort_values('no_recoreds', ascending=False)
 return missings
def prepare_data(t):
def f(d):
d=d.sort_values(by=['measurement_number'])
 return pd.DataFrame({
'lx':[ d['linear_acceleration_X'].values ],
'ly':[ d['linear_acceleration_Y'].values ],
'lz':[ d['linear_acceleration_Z'].values ],
'ax':[ d['angular_velocity_X'].values ],
'ay':[ d['angular_velocity_Y'].values ],
'az':[ d['angular_velocity_Z'].values ],
})
 t= t.groupby('series_id' ).apply(f)
 def mfft(x):
return [ x/math.sqrt(128.0)for x in np.absolute(np.fft.fft(x)) ][1:65]
t['lx_f']=[ mfft(x)for x in t['lx'].values ]
t['ly_f']=[ mfft(x)for x in t['ly'].values ]
t['lz_f']=[ mfft(x)for x in t['lz'].values ]
t['ax_f']=[ mfft(x)for x in t['ax'].values ]
t['ay_f']=[ mfft(x)for x in t['ay'].values ]
t['az_f']=[ mfft(x)for x in t['az'].values ]
return t
t=prepare_data(train_x)
 t=pd.merge(t,train_y[['series_id','surface','group_id']],on='series_id')
 t=t.rename(columns={"surface": "y"})
 def aggf(d, feature):
va= np.array(d[feature].tolist())
 mean= sum(va)/va.shape[0]
var= sum([(va[i,:]-mean)**2 for i in range(va.shape[0])])/va.shape[0]
dev= [ math.sqrt(x)for x in var ]
return pd.DataFrame({
'mean': [ mean ],
'dev' : [ dev ],
})
 display={
'hard_tiles_large_space':'r-.',
'concrete':'g-.',
'tiled':'b-.',
'fine_concrete':'r-',
'wood':'g-',
'carpet':'b-',
'soft_pvc':'y-',
'hard_tiles':'r--',
'soft_tiles':'g--',
}
plt.figure(figsize=(14, 8*7))
features=['lx_f','ly_f','lz_f','ax_f','ay_f','az_f']
count=0
for feature in features:
stat= t.groupby('y' ).apply(aggf,feature)
 stat.index= stat.index.droplevel(-1)
 b=[*range(len(stat.at['carpet','mean'])) ]
count+=1
plt.subplot(len(features)+1,1,count)
 for i,(k,v)in enumerate(display.items()):
plt.plot(b, stat.at[k,'mean'], v, label=k)
 leg = plt.legend(loc='best', ncol=3, mode="expand", shadow=True, fancybox=True)
 plt.title("sensor: " + feature)
 plt.xlabel("frequency component")
 plt.ylabel("amplitude")
 count+=1
plt.subplot(len(features)+1,1,count)
 k='concrete'
v=display[k]
feature='lz_f'
stat= t.groupby('y' ).apply(aggf,feature)
 stat.index= stat.index.droplevel(-1)
 b=[*range(len(stat.at['carpet','mean'])) ]
plt.errorbar(b, stat.at[k,'mean'], yerr=stat.at[k,'dev'], fmt=v)
 plt.title("sample for error bars(lz_f, surface concrete)")
 plt.xlabel("frequency component")
 plt.ylabel("amplitude")
 plt.show()<set_options>
df_missings = get_missing_features(df_train)
 print(df_missings )
del train_x, train_y
gc.collect()<import_modules>
for i, feature in enumerate(list(df_train.drop(['id'], axis=1))):
if df_train[feature].isnull().sum() > 0:
df_train[feature].fillna(df_train[feature].mode() [0],inplace=True)
 for i, feature in enumerate(list(df_test.drop(['id'], axis=1))):
if df_test[feature].isnull().sum() > 0:
df_test[feature].fillna(df_test[feature].mode() [0],inplace=True )
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import itertools
import math
from sklearn.metrics import *
from sklearn.model_selection import *
import keras
from keras.layers import *
from keras.callbacks import *
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
get_missing_features(df_train)
 get_missing_features(df_test )
X_train = pd.read_csv(PATH+"/X_train.csv")
 X_test = pd.read_csv(PATH+"/X_test.csv")
 y_train = pd.read_csv(PATH+"/y_train.csv")
 sub = pd.read_csv(PATH+"/sample_submission.csv" )<train_model>
cat_cols = [col for col in df_train.columns if '_cat' in col]
dummed_cols = []
for cat_col in cat_cols:
unique_values = len(np.unique(df_train[cat_col]))
if unique_values < 50:
dummed_cols.append(cat_col)
 print('{} has {} unique values'.format(cat_col, unique_values))
print("Train: measurements = %d , features = %d" % X_train.shape)
 X_train.head()<drop_column>
id_test = df_test['id'].values
y = df_train['target'].values
df_train = df_train.drop(['target','id'], axis = 1)
 df_test = df_test.drop(['id'], axis = 1)
 cat_features = [a for a in df_train.columns if a.endswith('cat')]
for column in cat_features:
temp = pd.get_dummies(pd.Series(df_train[column]))
df_train = pd.concat([df_train,temp],axis=1)
 df_train = df_train.drop([column],axis=1)
 for column in cat_features:
temp = pd.get_dummies(pd.Series(df_test[column]))
df_test = pd.concat([df_test,temp],axis=1)
 df_test = df_test.drop([column],axis=1)
 print(df_train.values.shape, df_test.values.shape )
X_train.drop(['row_id', "series_id", "measurement_number"], axis=1, inplace=True)
 X_train = X_train.values.reshape(( 3810, 128, 10))
X_test.drop(['row_id', "series_id", "measurement_number"], axis=1, inplace=True)
 X_test = X_test.values.reshape(( 3816, 128, 10))<define_variables>
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def gini_normalized(a, p):
return gini(a, p)/ gini(a, a )
encode_dic = {'fine_concrete': 0,
'concrete': 1,
'soft_tiles': 2,
'tiled': 3,
'soft_pvc': 4,
'hard_tiles_large_space': 5,
'carpet': 6,
'hard_tiles': 7,
'wood': 8}
decode_dic = {0: 'fine_concrete',
1: 'concrete',
2: 'soft_tiles',
3: 'tiled',
4: 'soft_pvc',
5: 'hard_tiles_large_space',
6: 'carpet',
7: 'hard_tiles',
8: 'wood'}
y_train = y_train['surface'].map(encode_dic ).astype(int )<choose_model_class>
class Create_ensemble(object):
def __init__(self, n_splits, base_models):
self.n_splits = n_splits
self.base_models = base_models
def predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=2016 ).split(X, y))
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, valid_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_valid = X[valid_idx]
y_valid = y[valid_idx]
clf.fit(X_train, y_train)
 valid_pred = clf.predict_proba(X_valid)[:,1]
S_train[valid_idx, i] = valid_pred
S_test_i[:, j] = clf.predict_proba(T)[:,1]
print("
Training Gini for model {} : {}".format(i, eval_gini(y, S_train[:,i])))
 S_test[:, i] = S_test_i.mean(axis=1)
 return S_train, S_test
class Attention(Layer):
def __init__(self, step_dim, W_regularizer=None, b_regularizer=None, W_constraint=None, b_constraint=None, bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],), initializer=self.init, name='{}_W'.format(self.name), regularizer=self.W_regularizer, constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],), initializer='zero', name='{}_b'.format(self.name), regularizer=self.b_regularizer, constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) , K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias: eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None: a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
lgb_params = {}
lgb_params['learning_rate'] = 0.02
lgb_params['n_estimators'] = 700
lgb_params['max_bin'] = 15
lgb_params['subsample'] = 0.8
lgb_params['subsample_freq'] = 10
lgb_params['colsample_bytree'] = 0.8
lgb_params['min_child_samples'] = 800
lgb_params['random_state'] = 99
lgb_params['scale_pos_weight'] = 3
lgb_params2 = {}
lgb_params2['learning_rate'] = 0.02
lgb_params2['n_estimators'] = 900
lgb_params2['max_bin'] = 20
lgb_params2['subsample'] = 0.8
lgb_params2['subsample_freq'] = 10
lgb_params2['colsample_bytree'] = 0.8
lgb_params2['min_child_samples'] = 600
lgb_params2['random_state'] = 99
lgb_params2['scale_pos_weight'] = 3
lgb_model = LGBMClassifier(**lgb_params)
 lgb_model2 = LGBMClassifier(**lgb_params2 )
def make_model() :
inp = Input(shape=(128, 10))
x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(inp)
 x = Attention(128 )(x)
 x = Dense(9, activation="softmax" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<train_model>
lgb_stack = Create_ensemble(n_splits = 5, base_models = [lgb_model, lgb_model2])
 X = df_train
Y = y
T = df_test
lgb_train_pred, lgb_test_pred = lgb_stack.predict(X, Y, T )
<compute_test_metric><EOS> 
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = lgb_test_pred.mean(axis=1)
 sub.to_csv('submission.csv', float_format='%.6f', index=False )