Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<train_on_grid>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn import metrics
from sklearn.metrics import roc_auc_score
from sklearn.utils import shuffle
from sklearn.model_selection import StratifiedKFold
import xgboost as xgb
reg = RidgeCV()
reg.fit(X, y)
 print("Best alpha using built-in RidgeCV: %f" % reg.alpha_)
 print("Best score using built-in RidgeCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("Ridge picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )<train_on_grid>
train = pd.read_csv(".. /input/train.csv", na_values='-1')
 test = pd.read_csv(".. /input/test.csv", na_values='-1')
 print(train.shape)
 print(test.shape )
reg = ElasticNetCV()
reg.fit(X, y)
 print("Best alpha using built-in ElasticNetCV: %f" % reg.alpha_)
 print("Best score using built-in ElasticNetCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("ElasticNet picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables")
 ridge_coef = coef[coef != 0]<train_on_grid>
desired_apriori=0.10
idx_0 = train[train.target == 0].index
idx_1 = train[train.target == 1].index
nb_0 = len(train.loc[idx_0])
 nb_1 = len(train.loc[idx_1])
 undersampling_rate =(( 1-desired_apriori)*nb_1)/(nb_0*desired_apriori)
 undersampled_nb_0 = int(undersampling_rate*nb_0)
 print('Rate to undersample records with target=0: {}'.format(undersampling_rate))
print('Number of records with target=0 after undersampling: {}'.format(undersampled_nb_0))
undersampled_idx = shuffle(idx_0, random_state=37, n_samples=undersampled_nb_0)
 idx_list = list(undersampled_idx)+ list(idx_1)
 train = train.loc[idx_list].reset_index(drop=True )
reg = LassoCV()
reg.fit(X, y)
 print("Best alpha using built-in LassoCV: %f" % reg.alpha_)
 print("Best score using built-in LassoCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )<choose_model_class>
vars_to_drop = ['ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_12_bin', 'ps_ind_13_bin', 'ps_reg_03', 'ps_car_03_cat',
'ps_car_05_cat','ps_car_14']
train.drop(vars_to_drop, inplace=True, axis=1)
 test.drop(vars_to_drop, inplace=True, axis=1)
 print(train.shape)
 print(test.shape )
models = [('DTR', DecisionTreeRegressor()), 
('RFR', RandomForestRegressor()), 
('KNR', KNeighborsRegressor()), 
('GBR', GradientBoostingRegressor()), 
('LR', LinearRegression()), 
('XGB', XGBRegressor()), 
('LGBM', LGBMRegressor()), 
('SVR', SVR()), 
('Ridge', Ridge(alpha=10)) , 
('Lasso', Lasso(alpha=0.003487)) , 
('ENet', ElasticNet(alpha=0.006974)) ] 
<compute_train_metric>
unwanted = train.columns[train.columns.str.startswith('ps_calc_')]
train = train.drop(unwanted, axis=1)
 test = test.drop(unwanted, axis=1 )
results = []
names = []
for coef in [ridge_coef.index, enet_coef.index, lasso_coef.index]:
for name, model in models:
kfold = KFold(n_splits=10, random_state=21)
 cv_results = cross_val_score(model, X[coef], y, cv=kfold, scoring='neg_mean_squared_error')
 results.append(cv_results)
 names.append(name)
 msg = "%s: %f(%f)" %(name, cv_results.mean() , cv_results.std())
 print(msg )<train_on_grid>
train_data_nato_median = pd.DataFrame()
for column in train.columns:
train_data_nato_median[column] = train[column].fillna(train[column].median())
 train = train_data_nato_median.copy()
test_data_nato_median = pd.DataFrame()
for column in test.columns:
test_data_nato_median[column] = test[column].fillna(test[column].median())
 test = test_data_nato_median.copy()
params = {
'alpha': [0.001]
}
reg = Lasso()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 lasso = rs.best_estimator_<train_on_grid>
column_names = train.columns
categorical_column = column_names[column_names.str[10] == 'c']
def int_to_categorical(data):

 for column in categorical_column:
data[column] = data[column].astype('category' )
params = {
'learning_rate': [0.01],
'n_estimators': [2000],
'max_depth': [11],
'min_samples_split': [200],
'min_samples_leaf': [10],
'max_features': ['sqrt'],
'subsample': [0.85]
}
reg = GradientBoostingRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 gbr = rs.best_estimator_<train_on_grid>
datas = [train,test]
for data in datas:
int_to_categorical(data)
 print(test.dtypes )
params = {
'alpha': [0.001]
}
reg = ElasticNet()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 elasticnet = rs.best_estimator_<train_on_grid>
def creating_dummies(data):

 for column in categorical_column:
dummies = pd.get_dummies(data[column],prefix=column)
 data = pd.concat([data,dummies],axis =1)
 data.drop([column],axis=1,inplace= True )
params = {
'learning_rate': [0.01],
'n_estimators': [3000],
'max_depth': [3],
'min_child_weight': [5],
'gamma': [0],
'colsample_bytree': [0.65],
'subsample': [0.6],
'reg_alpha':[1e-6]
}
reg = XGBRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 xgboost = rs.best_estimator_<train_on_grid>
for column in categorical_column:
dummies = pd.get_dummies(train[column],prefix=column)
 train = pd.concat([train,dummies],axis =1)
 train.drop([column],axis=1,inplace= True)
 for column in categorical_column:
dummies = pd.get_dummies(test[column],prefix=column)
 test = pd.concat([test,dummies],axis =1)
 test.drop([column],axis=1,inplace= True)
 print(train.shape)
 print(test.shape )
params = {
'learning_rate': [0.01],
'n_estimators': [3000],
'max_depth': [3],
'min_child_weight': [1],
'gamma': [0],
'colsample_bytree': [0.8],
'subsample': [0.6],
}
reg = LGBMRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_ )<compute_train_metric>
def gini(actual, pred, cmpcol = 0, sortcol = 1):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = gini_normalized(labels, preds)
 return 'gini', gini_score
level_0 = [('ENet',elasticnet),('GBR', gbr)]
level_1 = lasso
model = StackingRegressor(estimators=level_0, final_estimator=level_1, cv=10)
 cv = KFold(n_splits=10, random_state=21)
 scores = cross_val_score(model, X[lasso_coef.index],y,cv=cv,scoring='neg_mean_absolute_error')
 print(scores.mean() )<train_model>
kfold = 3
skf = StratifiedKFold(n_splits=kfold, random_state=42 )
model.fit(X[lasso_coef.index],y )<save_to_csv>
params = {
'min_child_weight': 10.0,
'objective': 'binary:logistic',
'max_depth': 7,
'max_delta_step': 1.8,
'colsample_bytree': 0.4,
'subsample': 0.8,
'eta': 0.025,
'gamma': 0.65,
'num_boost_round' : 700
}
pred_stacked = model.predict(scaled_test[lasso_coef.index])
 pred = np.expm1(pred_stacked)
 sub = test[['Id']]
sub['SalePrice'] = pred
sub[['Id', 'SalePrice']].to_csv('pred_submission.csv', index=False, encoding='utf-8' )<import_modules>
X = train.drop(['id', 'target'], axis=1 ).values
y = train.target.values
test_id = test.id.values
test = test.drop('id', axis=1 ).values
import matplotlib
import matplotlib.pyplot as plt
from scipy.stats import skew
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from itertools import product
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error<load_from_csv>
X = X[:,[142,135,150,14,138,143,139,141,136,61,151,11,179,146,140,86,175,147, 152,145,180,148,177,133,59,60,183,14,51,34,67,137,16,161,88,85,159,33,178,184,65,13,12,63,18,156,83,87]]
test = test[:,[142,135,150,14,138,143,139,141,136,61,151,11,179,146,140,86,175,147, 152,145,180,148,177,133,59,60,183,14,51,34,67,137,16,161,88,85,159,33,178,184,65,13,12,63,18,156,83,87]]
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
sub = pd.DataFrame()
sub['id'] = test_id
sub['target'] = np.zeros_like(test_id )
<define_variables>
for i,(train_index, test_index)in enumerate(skf.split(X, y)) :
print('[Fold %d/%d]' %(i + 1, kfold))
X_train, X_valid = X[train_index], X[test_index]
y_train, y_valid = y[train_index], y[test_index]
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 d_test = xgb.DMatrix(test)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
mdl = xgb.train(params, d_train, 1600, watchlist, early_stopping_rounds=70, feval=gini_xgb, maximize=True, verbose_eval=100)
 print('[Fold %d/%d Prediciton:]' %(i + 1, kfold))
p_test = mdl.predict(d_test, ntree_limit=mdl.best_ntree_limit)
 sub['target'] += p_test/kfold
<concatenate><EOS> 
sub.to_csv('StratifiedKFold5.csv', index=False )