Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<load_from_csv>
sample_submission = pd.read_csv(".. /input/cat-in-the-dat-ii/sample_submission.csv")
 test = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv")
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
train["nom_0by1"]=train.nom_0.str.cat(train['nom_1'])
 train["nom_0by2"]=train.nom_0.str.cat(train['nom_2'])
 train["nom_0by3"]=train.nom_0.str.cat(train['nom_3'])
 train["nom_0by4"]=train.nom_0.str.cat(train['nom_4'])
 test["nom_0by1"]=test.nom_0.str.cat(test['nom_1'])
 test["nom_0by2"]=test.nom_0.str.cat(test['nom_2'])
 test["nom_0by3"]=test.nom_0.str.cat(test['nom_3'])
 test["nom_0by4"]=test.nom_0.str.cat(test['nom_4'])
 train["nom_1by2"]=train.nom_1.str.cat(train['nom_2'])
 train["nom_1by3"]=train.nom_1.str.cat(train['nom_3'])
 train["nom_1by4"]=train.nom_1.str.cat(train['nom_4'])
 test["nom_1by2"]=test.nom_1.str.cat(test['nom_2'])
 test["nom_1by3"]=test.nom_1.str.cat(test['nom_3'])
 test["nom_1by4"]=test.nom_1.str.cat(test['nom_4'])
 train["nom_2by3"]=train.nom_2.str.cat(train['nom_3'])
 train["nom_2by4"]=train.nom_2.str.cat(train['nom_4'])
 test["nom_2by3"]=test.nom_2.str.cat(test['nom_3'])
 test["nom_2by4"]=test.nom_2.str.cat(test['nom_4'])
 train["nom_3by4"]=train.nom_3.str.cat(train['nom_4'])
 test["nom_3by4"]=test.nom_3.str.cat(test['nom_4'])
 train["bin_3by4"]=train.bin_3.str.cat(train['bin_4'])
 test["bin_3by4"]=test.bin_3.str.cat(test['bin_4'])
 train["bin_3bynom_0"]=train.bin_3.str.cat(train['nom_0'])
 train["bin_3bynom_1"]=train.bin_3.str.cat(train['nom_1'])
 train["bin_3bynom_2"]=train.bin_3.str.cat(train['nom_2'])
 train["bin_3bynom_3"]=train.bin_3.str.cat(train['nom_3'])
 train["bin_3bynom_4"]=train.bin_3.str.cat(train['nom_4'])
 test["bin_3bynom_0"]=test.bin_3.str.cat(test['nom_0'])
 test["bin_3bynom_1"]=test.bin_3.str.cat(test['nom_1'])
 test["bin_3bynom_2"]=test.bin_3.str.cat(test['nom_2'])
 test["bin_3bynom_3"]=test.bin_3.str.cat(test['nom_3'])
 test["bin_3bynom_4"]=test.bin_3.str.cat(test['nom_4'])
 train["bin_4bynom_0"]=train.bin_4.str.cat(train['nom_0'])
 train["bin_4bynom_1"]=train.bin_4.str.cat(train['nom_1'])
 train["bin_4bynom_2"]=train.bin_4.str.cat(train['nom_2'])
 train["bin_4bynom_3"]=train.bin_4.str.cat(train['nom_3'])
 train["bin_4bynom_4"]=train.bin_4.str.cat(train['nom_4'])
 test["bin_4bynom_0"]=test.bin_4.str.cat(test['nom_0'])
 test["bin_4bynom_1"]=test.bin_4.str.cat(test['nom_1'])
 test["bin_4bynom_2"]=test.bin_4.str.cat(test['nom_2'])
 test["bin_4bynom_3"]=test.bin_4.str.cat(test['nom_3'])
 test["bin_4bynom_4"]=test.bin_4.str.cat(test['nom_4'])
 train["nom_5bynom_8"]=train.nom_5.str.cat(train['nom_8'])
 test["nom_5bynom_8"]=test.nom_5.str.cat(test['nom_8'])
 train["nom_6bynom_9"]=train.nom_6.str.cat(train['nom_9'])
 test["nom_6bynom_9"]=test.nom_6.str.cat(test['nom_9'])
raws_features = train_data.columns[2:]
train_X, train_y = train_data[raws_features], train_data['target']
test_X = test_data[raws_features]<define_variables>
train['day_obj']=train.day.astype('str')
 test['day_obj']=test.day.astype('str')
 train['month_obj']=train.month.astype('str')
 test['month_obj']=test.month.astype('str')
 train['ord_0_obj']=train.ord_0
train['ord_1_obj']=train.ord_1
train['ord_2_obj']=train.ord_2
train['ord_3_obj']=train.ord_3
train['ord_4_obj']=train.ord_4
train['ord_5_obj']=train.ord_5
test['ord_0_obj']=test.ord_0
test['ord_1_obj']=test.ord_1
test['ord_2_obj']=test.ord_2
test['ord_3_obj']=test.ord_3
test['ord_4_obj']=test.ord_4
test['ord_5_obj']=test.ord_5
train['nom_0_obj']=train.nom_0
train['nom_1_obj']=train.nom_1
train['nom_2_obj']=train.nom_2
train['nom_3_obj']=train.nom_3
train['nom_4_obj']=train.nom_4
test['nom_0_obj']=test.nom_0
test['nom_1_obj']=test.nom_1
test['nom_2_obj']=test.nom_2
test['nom_3_obj']=test.nom_3
test['nom_4_obj']=test.nom_4
train['bin_0_obj']=train.bin_0
train['bin_1_obj']=train.bin_1
train['bin_2_obj']=train.bin_2
train['bin_3_obj']=train.bin_3
train['bin_4_obj']=train.bin_4
test['bin_0_obj']=test.bin_0
test['bin_1_obj']=test.bin_1
test['bin_2_obj']=test.bin_2
test['bin_3_obj']=test.bin_3
test['bin_4_obj']=test.bin_4
n_splits = 10
num_round = 77777
seed = 7777<train_on_grid>
train["daybymonth"]=train.month_obj.str.cat(train['day_obj'])
 test["daybymonth"]=test.month_obj.str.cat(test['day_obj'] )
class CVClassifier() :
def __init__(self, estimator, n_splits=5, stratified=True, num_round=77777, **params):
self.n_splits_ = n_splits
self.scores_ = []
self.clf_list_ = []
self.estimator_ = estimator
self.stratified_ = stratified
self.num_round_ = num_round
if params:
self.params_ = params
def cv(self, train_X, train_y):
if self.stratified_:
folds = StratifiedKFold(self.n_splits_, shuffle=True, random_state=seed)
 else:
folds = KFold(self.n_splits_, shuffle=True, random_state=seed)
 oof = np.zeros(len(train_y))
for fold,(train_idx, val_idx)in enumerate(folds.split(train_X, train_y)) :
print('fold %d' % fold)
 trn_data, trn_y = train_X.iloc[train_idx], train_y[train_idx]
val_data, val_y = train_X.iloc[val_idx], train_y[val_idx]
if self.estimator_ == 'lgbm':
train_set = lgb.Dataset(data=trn_data, label=trn_y)
 val_set = lgb.Dataset(data=val_data, label=val_y)
 clf = lgb.train(params=params, train_set=train_set, num_boost_round=num_round,
valid_sets=[train_set, val_set], verbose_eval=100, early_stopping_rounds=200)
 oof[val_idx] = clf.predict(train_X.iloc[val_idx], num_iteration=clf.best_iteration)
 elif self.estimator_ == 'xgb':
train_set = xgb.DMatrix(data=trn_data, label=trn_y)
 val_set = xgb.DMatrix(data=val_data, label=val_y)
 watchlist = [(train_set, 'train'),(val_set, 'valid')]
clf = xgb.train(self.params_, train_set, self.num_round_, watchlist,
early_stopping_rounds=200, verbose_eval=100)
 oof[val_idx] = clf.predict(val_set, ntree_limit=clf.best_ntree_limit)
 elif self.estimator_ == 'cat':
clf = CatBoostClassifier(self.num_round_, task_type='GPU', early_stopping_rounds=500, **self.params_)
 clf.fit(trn_data, trn_y, eval_set=(val_data, val_y), cat_features=[], use_best_model=True, verbose=500)
 oof[val_idx] = clf.predict_proba(val_data)[:, 1]
else:
clf = self.estimator_.fit(trn_data, trn_y)
 try:
oof[val_idx] = clf.predict_proba(val_data)[:, 1]
except AttributeError:
oof[val_idx] = clf.decision_function(val_data)
 self.clf_list_.append(clf)
 fold_score = roc_auc_score(train_y[val_idx], oof[val_idx])
 self.scores_.append(fold_score)
 print('Fold score: {:<8.5f}'.format(fold_score))
self.oof_ = oof
self.score_ = roc_auc_score(train_y, oof)
 print("CV score: {:<8.5f}".format(self.score_))
def predict(self, test_X):
self.predictions_ = np.zeros(len(test_X))
if self.estimator_ == 'lgbm':
self.feature_importance_df_ = pd.DataFrame()
for fold, clf in enumerate(self.clf_list_):
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold + 1
self.feature_importance_df_ = pd.concat([self.feature_importance_df_, fold_importance_df], axis=0)
 self.predictions_ += clf.predict(test_X, num_iteration=clf.best_iteration)*(self.scores_[fold] / sum(self.scores_))
elif self.estimator_ == 'xgb':
for fold, clf in enumerate(self.clf_list_):
self.predictions_ += clf.predict(xgb.DMatrix(test_X), ntree_limit=clf.best_ntree_limit)\
*(self.scores_[fold] / sum(self.scores_))
elif self.estimator_ == 'cat':
for fold, clf in enumerate(self.clf_list_):
self.predictions_ += clf.predict_proba(test_X)[:, 1] *(self.scores_[fold] / sum(self.scores_))
else:
for fold, clf in enumerate(self.clf_list_):
self.predictions_ += clf.predict_proba(test_X)[:, 1] *(self.scores_[fold] / sum(self.scores_))<choose_model_class>
cat_cols=['nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj','ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj',
'ord_5_obj',
'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj',
'bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj',
"nom_3by4","bin_4bynom_0"]
for c in cat_cols:
data_tmp=pd.DataFrame({c:train[c],'target':train.target})
 target_mean=data_tmp.groupby(c)['target'].mean()
test[c]=test[c].map(target_mean)
 tmp=np.repeat(np.nan, train.shape[0])
 kf=KFold(n_splits=4, shuffle=True, random_state=72)
 for idx_1,idx_2 in kf.split(train):
target_mean=data_tmp.iloc[idx_1].groupby(c)['target'].mean()
tmp[idx_2]=train[c].iloc[idx_2].map(target_mean)
 train[c]=tmp
class CVForBO() :
def __init__(self, model, train_X, train_y, test_X, base_params, int_params=[], n_splits=5, num_round=77777):
self.oofs_ = []
self.params_ = []
self.predictions_ = []
self.cv_scores_ = []
self.model_ = model
self.train_X_ = train_X
self.train_y_ = train_y
self.test_X_ = test_X
self.base_params_ = base_params
self.int_params_ = int_params
self.n_splits_ = n_splits
self.num_round_ = num_round
def cv(self, **opt_params):
for p in self.int_params_:
if p in opt_params:
opt_params[p] = int(np.round(opt_params[p]))
self.base_params_.update(opt_params)
 cv_model = CVClassifier(self.model_, n_splits=self.n_splits_, num_round=self.num_round_, **self.base_params_)
 cv_model.cv(self.train_X_, self.train_y_)
 cv_model.predict(self.test_X_)
 self.oofs_.append(cv_model.oof_)
 self.predictions_.append(cv_model.predictions_)
 self.params_.append(self.base_params_)
 self.cv_scores_.append(cv_model.score_)
 return cv_model.score_
def post_process(self, model_type=None, oof_path='inter_oofs.csv', pred_path='inter_preds.csv', params_path='inter_params.csv'):
if not model_type:
model_type=self.model_
cols = ['{}_{}_{}'.format(model_type, str(self.cv_scores_[k] ).split('.')[-1][:5], k)for k in range(len(self.cv_scores_)) ]
self.oof_df = pd.DataFrame(np.array(self.oofs_ ).T, columns=cols)
 self.pred_df = pd.DataFrame(np.array(self.predictions_ ).T, columns=cols)
 self.params_df = pd.DataFrame(self.params_ ).T.rename(columns={c_old: c_new for c_old, c_new in enumerate(cols)})
 self.oof_df.to_csv(oof_path)
 self.pred_df.to_csv(pred_path)
 self.params_df.to_csv(params_path )<choose_model_class>
train['nom_mean']=(train.nom_5+train.nom_6+train.nom_7)/3
test['nom_mean']=(test.nom_5+test.nom_6+test.nom_7)/3
cat_params = {
'eval_metric': 'AUC',
'bootstrap_type': 'Bernoulli',
'objective': 'Logloss',
'od_type': 'Iter',
'random_seed': seed,
'allow_writing_files': False}
cv_cat_for_BO = CVForBO('cat', train_X, train_y, test_X, cat_params, ['depth'])
 cat_BO = BayesianOptimization(cv_cat_for_BO.cv, {
'depth':(2, 4),
'l2_leaf_reg':(37, 97),
'random_strength':(5, 17),
'eta':(0.01, 0.1)
 }, random_state=seed)
 cat_BO.maximize(init_points=2, n_iter=15, acq='ei' )<save_to_csv>
cols_to_use = ['ord_1', 'ord_2', 'ord_3', 'ord_4','ord_5','nom_0','nom_1','nom_2','nom_3','nom_4','bin_3','bin_4']
num_cols_to_use = ['ord_0','bin_0','bin_1','bin_2','day','month','nom_5','nom_6','nom_7','nom_8','nom_9','day_obj','month_obj',
'ord_0_obj','ord_1_obj','ord_2_obj','ord_3_obj','ord_4_obj','ord_5_obj',
'nom_0_obj','nom_1_obj','nom_2_obj','nom_3_obj','nom_4_obj',
'bin_0_obj','bin_1_obj','bin_2_obj','bin_3_obj','bin_4_obj',
"nom_3by4","bin_4bynom_0"
]
cols_sum = cols_to_use + num_cols_to_use
X = train[cols_sum]
y = train.target
train_X,val_X,train_y,val_y = train_test_split(X,y,random_state = 0)
 X_val = test[cols_sum]
numerical_transformer = SimpleImputer(strategy='mean')
 categorical_transformer = Pipeline(steps=[ 
('imputer',SimpleImputer(strategy='constant')) , 
('onehot',OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, num_cols_to_use), 
('cat',categorical_transformer, cols_to_use)
 ])
 my_model = CatBoost({'loss_function':'Logloss','num_boost_round':2500,'learning_rate':0.03,'early_stopping_rounds': 10,'depth':2})
 my_pipeline = Pipeline(steps=[('preprocessor',preprocessor), 
('model',my_model)
 ])
 my_pipeline.fit(train_X,train_y)
 prediction = my_pipeline.predict(X_val )
max_idx = cv_cat_for_BO.cv_scores_.index(cat_BO.max['target'])
 sub_df = pd.DataFrame({'ID_code': test_data['ID_code'],
'target': cv_cat_for_BO.predictions_[max_idx]})
 sub_df.to_csv('submissions.csv', index=False )<import_modules>
val_predictions = my_pipeline.predict(val_X)
 print(roc_auc_score(val_y,val_predictions))
<load_from_csv><EOS> 
ss=sample_submission
ss.target = prediction
ss.to_csv("submission.csv", index=False )