Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<categorify>
pd.set_option('display.max_columns', 100 )
train_fe_X.to_feather('Train_FE')
 test_fe_X.to_feather('Test_FE')
 train_df_Y.to_feather('Target')
<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train_fe_X.to_feather('Train_FE')
 test_fe_X.to_feather('Test_FE')
 train_df_Y.to_feather('Target')
<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
<define_variables>
data = []
for feature in train.columns:
if feature == 'id':
use = 'id'
elif feature == 'target':
use = 'target'
else:
use = 'input'
if 'bin' in feature or feature == 'target':
type = 'binary'
elif 'cat' in feature or feature == 'id':
type = 'categorical'
elif train[feature].dtype == float or isinstance(train[feature].dtype,float):
type = 'real'
elif train[feature].dtype == int:
type = 'integer'
preserve = True
if feature =='id':
preserve = False
dtype = train[feature].dtype
category = 'none'
if 'ind' in feature:
category = 'individual'
elif 'reg' in feature:
category = 'registration'
elif 'car' in feature:
category = 'car'
elif 'calc' in feature:
category = 'calculated'
feature_dict = {
'var_name':feature,
'use':use,
'type':type,
'preserve':preserve,
'dtype':dtype,
'category':category
}
data.append(feature_dict)
 metadata = pd.DataFrame(data,columns=['var_name', 'use', 'type', 'preserve', 'dtype', 'category'])
 metadata.set_index('var_name',inplace=True )
<load_from_csv>
metadata[(metadata['type']=='categorical')&(metadata.preserve)]
x_df = pd.read_csv('.. /input/X_train.csv')
 x_test_df = pd.read_csv('.. /input/X_test.csv')
 y_df = pd.read_csv('.. /input/y_train.csv')
 x_df.head()<count_values>
pd.DataFrame({'count':metadata.groupby('category')['category'].size() } ).reset_index()
len(y_df['series_id'].value_counts() )<count_values>
pd.DataFrame({'count':metadata.groupby(['use','type'])['use'].size() } ).reset_index()
len(x_df['series_id'].value_counts() )<count_values>
desired_apriori=0.10
idx_0 = train[train.target == 0].index
idx_1 = train[train.target == 1].index
no_0 = len(train.loc[idx_0])
 no_1 = len(train.loc[idx_1])
 undersampling_rate =(( 1-desired_apriori)*no_1)/(desired_apriori*no_0)
 undersampled_no_0 = int(undersampling_rate*no_0)
 print('Rate to undersample records with target=0: {}'.format(undersampling_rate))
print('Number of records with target=0 after undersampling: {}'.format(undersampled_no_0))
undersampled_idx = shuffle(idx_0,random_state=242,n_samples=undersampled_no_0)
 idx_list = list(undersampled_idx)+list(idx_1)
 train = train.loc[idx_list].reset_index(drop=True )
len(x_df['measurement_number'].value_counts() )<count_missing_values>
df_null =(train == -1 ).sum().sort_values(ascending=False ).reset_index()
df_null.columns = ['features','count']
df_null = df_null[df_null['count'] > 0]
df_null
print(y_df.head())
 print(x_df.isnull().values.any())
 print(x_test_df.isnull().values.any())
 print(y_df.isnull().values.any() )<compute_test_metric>
metadata.loc[col_to_drop,'preserve']
def phi(x):
return math.atan2(x['orientation_W']*x['orientation_X'] + x['orientation_Y']*x['orientation_Z'], 1-2*(x['orientation_X']**2+x['orientation_Y']**2))
def theta(x):
return math.asin(2*(x['orientation_W']*x['orientation_Y'] - x['orientation_Z']*x['orientation_X']))
def chi(x):
return math.atan2(2*(x['orientation_W']*x['orientation_Z']+x['orientation_X']*x['orientation_Y']), 1-2*(x['orientation_Y']**2+x['orientation_Z']**2))<feature_engineering>
col_to_drop = train.columns[train.columns.str.startswith('ps_calc_')]
metadata.loc[col_to_drop,'preserve'] = False
train = train.drop(col_to_drop,axis=1)
 test = test.drop(col_to_drop,axis=1 )
x_df['phi'] = x_df.apply(phi, axis=1)
 x_df['theta'] = x_df.apply(theta, axis=1)
 x_df['chi'] = x_df.apply(chi, axis=1)
 x_test_df['phi'] = x_test_df.apply(phi, axis=1)
 x_test_df['theta'] = x_test_df.apply(theta, axis=1)
 x_test_df['chi'] = x_test_df.apply(chi, axis=1)
 x_df.head()<feature_engineering>
vars_to_drop = ['ps_car_03_cat','ps_car_05_cat']
train = train.drop(vars_to_drop,axis=1)
 test = test.drop(vars_to_drop,axis=1)
 metadata.loc[(vars_to_drop),'preserve'] = False
def normalize_quaternions(data):
data['norm_quat'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2 + data['orientation_W']**2)
 data['mod_quat'] =(data['norm_quat'])**0.5
data['norm_X'] = data['orientation_X'] / data['mod_quat']
data['norm_Y'] = data['orientation_Y'] / data['mod_quat']
data['norm_Z'] = data['orientation_Z'] / data['mod_quat']
data['norm_W'] = data['orientation_W'] / data['mod_quat']
return data<normalization>
dummy = metadata.loc[df_null.features,:]
dummy[(dummy.type=='real')|(dummy.type=='integer')]
x_df = normalize_quaternions(x_df)
 x_test_df = normalize_quaternions(x_test_df)
 x_df.head()<feature_engineering>
mean_imputer = Imputer(missing_values=-1,strategy='mean',axis=0)
 dummy = metadata.loc[df_null.features,:]
real_int_cols = dummy[(dummy.type=='real')|(dummy.type=='integer')].index
for col in real_int_cols:
if metadata.loc[col].type =='real':
train[col] = mean_imputer.fit_transform(pd.DataFrame(train[col])).ravel()
train[col] = train[col].astype(float)
 test[col] = mean_imputer.transform(pd.DataFrame(test[col])).ravel()
test[col] = test[col].astype(float)
 if metadata.loc[col].type == 'integer':
train[col] = mean_imputer.fit_transform(pd.DataFrame(train[col])).ravel()
train[col] = train[col].astype(float)
 test[col] = mean_imputer.transform(pd.DataFrame(test[col])).ravel()
test[col] = test[col].astype(float )
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<categorify>
frequent_imputer = Imputer(missing_values=-1,strategy='most_frequent',axis=0)
 droped_cats = dummy[dummy.type=='categorical'].index[~dummy[dummy.type=='categorical'].index.isin(['ps_car_03_cat','ps_car_05_cat'])]
for col in droped_cats:
train[col] = frequent_imputer.fit_transform(pd.DataFrame(train[col])).ravel()
train[col] = train[col].astype(int)
 test[col] = frequent_imputer.fit_transform(pd.DataFrame(test[col])).ravel()
test[col] = test[col].astype(int )
x_df = feat_eng(x_df)
 x_test_df = feat_eng(x_test_df)
 x_df=x_df.reset_index()
x_test_df=x_test_df.reset_index()
print(x_df.shape)
 x_df.head()<sort_values>
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_tst_series, noise_level )
correlations = x_df.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.tail(20 )<categorify>
train_encoded, test_encoded = target_encode(train['ps_car_11_cat'],
test['ps_car_11_cat'],
target=train.target,
min_samples_leaf=100,
smoothing=10,
noise_level=0.01)
 train['ps_car_11_cat_te'] = train_encoded
train.drop('ps_car_11_cat',axis=1,inplace=True)
 metadata.loc['ps_car_11_cat', 'preserve'] =False
temp_dict = metadata.loc['ps_car_11_cat',:].to_dict()
temp_dict['var_name'] = 'ps_car_11_cat_te'
temp_dict['type'] ='real'
temp_dict['preserve'] = True
temp_dict['dtype'] = train['ps_car_11_cat_te'].dtype
test['ps_car_11_cat_te'] = test_encoded
test.drop('ps_car_11_cat',axis=1,inplace=True)
 metadata = pd.concat([metadata,pd.DataFrame(temp_dict,index=range(1)).set_index('var_name')] )
le = preprocessing.LabelEncoder()
y_df['surface'] = le.fit_transform(y_df['surface'])
 y_df['surface'].value_counts()<define_variables>
cat_features = metadata[(metadata.type=='categorical')&(metadata.preserve)].index
for col in cat_features:
temp = pd.get_dummies(pd.Series(train[col]),prefix=col)
 train = pd.concat([train,temp],axis=1)
 train.drop(col,inplace=True,axis=1)
 for col in cat_features:
temp = pd.get_dummies(pd.Series(test[col]),prefix=col)
 test = pd.concat([test,temp],axis=1)
 test.drop(col,inplace=True,axis=1 )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=59)
 predicted = np.zeros(( x_test_df.shape[0],9))
measured= np.zeros(( x_df.shape[0]))
score = 0<train_on_grid>
real_cols = metadata[(metadata.type=='real')&(metadata.preserve)].index
def run_gridsearch(X, y, clf, param_grid, cv=5):
grid_search = GridSearchCV(clf, param_grid=param_grid, cv=cv, verbose=2)
 start = time()
grid_search.fit(X, y)
 print(( "
GridSearchCV took {:.2f} seconds for {:d} candidate"
"parameter settings." ).format(time() - start, len(grid_search.cv_results_)))
 top_params = report(grid_search.cv_results_, 2)
 print(grid_search.best_params_)
 return top_params
def report(grid_scores, n_top=2):
top_scores = sorted(grid_scores, key=itemgetter(1), reverse=True)[:n_top]
print(grid_scores['mean_train_score'])
 return top_scores<define_search_space>
poly = PolynomialFeatures(degree=2,interaction_only=True,include_bias=False)
 new_poly_df_train = pd.DataFrame(poly.fit_transform(train[real_cols]),columns=poly.get_feature_names(real_cols))
new_poly_df_test = pd.DataFrame(poly.transform(train[real_cols]),columns=poly.get_feature_names(real_cols))
new_poly_df_train = new_poly_df_train.drop(real_cols,axis=1)
 new_poly_df_test =new_poly_df_test.drop(real_cols,axis=1)
 train = pd.concat([train,new_poly_df_train],axis=1)
 test = pd.concat([test,new_poly_df_train],axis=1 )
warnings.filterwarnings("ignore")
 print("-- Grid Parameter Search via 10-fold CV")

param_grid = {"loss": ["deviance"],
"n_estimators": [500],
"min_samples_split": [5, 20],
"max_depth": [10, 50],
"min_samples_leaf": [10, 50],
"max_leaf_nodes": [20, 40]
}
seed=43
clf = GradientBoostingClassifier(random_state=seed)
<set_options>
X = train.drop(['id','target'],axis=1)
 train_labels = X.columns
y = train['target']
X_test = test.drop('id',axis=1)
 y_test = np.zeros(X_test.shape[0])
 sub = test['id'].to_frame()
sub['target'] = 0
gc.collect()<split>
def gini(y, pred):
g = np.asarray(np.c_[y, pred, np.arange(len(y)) ], dtype=np.float)
 g = g[np.lexsort(( g[:,2], -1*g[:,1])) ]
gs = g[:,0].cumsum().sum() / g[:,0].sum()
gs -=(len(y)+ 1)/ 2.
 return gs / len(y)
 def gini_xgb(pred, y):
y = y.get_label()
return 'gini', gini(y, pred)/ gini(y, y )
X_train, X_test, y_train, y_test = ms.train_test_split(x_df, y_df['surface'],
test_size=0.25,
random_state=seed,
stratify= y_df['surface'] )<choose_model_class>
XgbC = XGBClassifier(
 learning_rate =0.1,
n_estimators=1000,
max_depth=5,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 xgb_params = XgbC.get_xgb_params()
xgtrain = xgboost.DMatrix(X.values,label=y.values)
 cvresult = xgboost.cv(xgb_params,xgtrain,num_boost_round=XgbC.get_params() ['n_estimators'],nfold=5,metrics='auc',early_stopping_rounds=50)
 print(cvresult.shape)
 XgbC.set_params(n_estimators=cvresult.shape[0])
 XgbC.fit(X, y,eval_metric='auc')
 dtrain_predictions = XgbC.predict(X)
 dtrain_predprob = XgbC.predict_proba(X)[:,1]
print("
Model Report")
 print("Accuracy : %.4g" % metrics.accuracy_score(y, dtrain_predictions))
print("AUC Score(Train): %f" % metrics.roc_auc_score(y, dtrain_predprob))
clf=RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(x_df, y_df['surface'])
 trn_acc = accuracy_score(y_train, clf.predict(X_train))
tst_acc = accuracy_score(y_test, clf.predict(X_test))
print('training accuracy:', trn_acc)
 print('test accuracy :', tst_acc )<predict_on_test>
XgbC = XGBClassifier(
 learning_rate =0.1,
n_estimators=cvresult.shape[0],
max_depth=5,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 xgb_params = XgbC.get_xgb_params()
KFOLD = StratifiedKFold(n_splits=5,random_state=12)
 X = X.values
for i,(train_index,valid_index)in enumerate(KFOLD.split(X,y)) :
X_train, X_valid = X[train_index],X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
d_train = xgboost.DMatrix(X_train,label=y_train)
 d_valid = xgboost.DMatrix(X_valid,label=y_valid)
 watchlist = [(d_train,'train'),(d_valid,'valid')]
xgboost_model = xgboost.train(xgb_params,d_train,cvresult.shape[0],watchlist,early_stopping_rounds=50,feval=gini_xgb,maximize=True,verbose_eval=100)
 sub['target'] +=xgboost_model.predict(xgboost.DMatrix(X_test.values),ntree_limit=xgboost_model.best_ntree_limit+50)/ 5
<categorify><EOS> 
submission = pd.DataFrame()
submission['id'] = sub['id']
submission['target'] = sub['target']
submission.to_csv('stacked.csv', index=False )