Kaggle competition: Categorical Feature Encoding Challenge II
train_columns=train_x.columns
evescore=pd.DataFrame({'training':[0],'valid_1':[0]},index=['rmse'])
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_x,target)) :
trn_data = lgb.Dataset(train_x.iloc[trn_idx][train_columns], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_x.iloc[val_idx][train_columns], label=target.iloc[val_idx])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000,early_stopping_rounds = 300)
 oof[val_idx] = clf.predict(train_x.iloc[val_idx][train_columns], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 evescore=evescore+pd.DataFrame(clf.best_score)
 predictions += clf.predict(test_df[train_columns], num_iteration=clf.best_iteration )<save_to_csv>
import numpy as np
import pandas as pd
from time import time
import pprint
import joblib
from catboost import CatBoostClassifier, Pool
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.metrics import make_scorer
predictions/5
lvl_one_train=pd.DataFrame(train_id)
 lvl_one_train['target']=oof
lvl_one_test=pd.DataFrame(test_id)
 lvl_one_test['target']=predictions
lvl_one_train.to_csv('lgb_train1.csv',index=False)
 lvl_one_test.to_csv('lgb_test1.csv',index=False )<import_modules>
SEED = 42
FOLDS = 10
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED )
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, cross_val_score 
<load_from_csv>
X = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv")
 Xt = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv")
 y = X.target.values
id_train = X.id
id_test = Xt.id
X.drop(['id', 'target'], axis=1, inplace=True)
 Xt.drop(['id'], axis=1, inplace=True)
 binary_vars = [c for c in X.columns if 'bin_' in c]
nominal_vars = [c for c in X.columns if 'nom_' in c]
high_cardinality = [c for c in nominal_vars if len(X[c].unique())> 16]
low_cardinality = [c for c in nominal_vars if len(X[c].unique())<= 16]
ordinal_vars = [c for c in X.columns if 'ord_' in c]
time_vars = ['day', 'month']
train = pd.read_csv('.. /input/train.csv' ).set_index('ID_code')
 print(train.shape)
 train.head() 
<load_from_csv>
X['ord_5_1'] = X['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 X['ord_5_2'] = X['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 Xt['ord_5_1'] = Xt['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 Xt['ord_5_2'] = Xt['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 ordinal_vars += ['ord_5_1', 'ord_5_2']
test = pd.read_csv('.. /input/test.csv' ).set_index('ID_code')
 print(test.shape)
 test.head()<choose_model_class>
ordinals = {
'ord_1' : {
'Novice' : 0,
'Contributor' : 1,
'Expert' : 2,
'Master' : 3,
'Grandmaster' : 4
},
'ord_2' : {
'Freezing' : 0,
'Cold' : 1,
'Warm' : 2,
'Hot' : 3,
'Boiling Hot' : 4,
'Lava Hot' : 5
}
}
def return_order(X, Xt, var_name):
mode = X[var_name].mode() [0]
el = sorted(set(X[var_name].fillna(mode ).unique())|set(Xt[var_name].fillna(mode ).unique()))
return {v:e for e, v in enumerate(el)}
for mapped_var in ordinal_vars:
if mapped_var not in ordinals:
mapped_values = return_order(X, Xt, mapped_var)
 X[mapped_var + '_num'] = X[mapped_var].replace(mapped_values)
 Xt[mapped_var + '_num'] = Xt[mapped_var].replace(mapped_values)
 else:
X[mapped_var + '_num'] = X[mapped_var].replace(ordinals[mapped_var])
 Xt[mapped_var + '_num'] = Xt[mapped_var].replace(ordinals[mapped_var] )
RS = 2019<prepare_x_and_y>
label_encoders = [LabelEncoder() for _ in range(X.shape[1])]
for col, column in enumerate(X.columns):
unique_values = pd.Series(X[column].append(Xt[column] ).unique())
 unique_values = unique_values[unique_values.notnull() ]
label_encoders[col].fit(unique_values)
 X.loc[X[column].notnull() , column] = label_encoders[col].transform(X.loc[X[column].notnull() , column])
 Xt.loc[Xt[column].notnull() , column] = label_encoders[col].transform(Xt.loc[Xt[column].notnull() , column] )
X_train,y_train = train.iloc[:,1:],train.iloc[:,0] 
<choose_model_class>
X = X.fillna(-1)
 Xt = Xt.fillna(-1 )
skf = StratifiedKFold(n_splits=5,random_state=RS,shuffle=True, )<compute_train_metric>
def frequency_encoding(column, df, df_test=None):
frequencies = df[column].value_counts().reset_index()
df_values = df[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).iloc[:,-1].values
if df_test is not None:
df_test_values = df_test[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).fillna(1 ).iloc[:,-1].values
else:
df_test_values = None
return df_values, df_test_values
for column in X.columns:
train_values, test_values = frequency_encoding(column, X, Xt)
 X[column+'_counts'] = train_values
Xt[column+'_counts'] = test_values
model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=4, learning_rate=0.05, n_estimators=4500,
objective='binary',min_child_samples=2, subsample=.9, colsample_bytree=.05,
reg_alpha=0.0, reg_lambda=0.0, seed=RS,importance_type='gain',)
 out = cross_val_score(model, X=X_train, y=y_train, scoring='roc_auc', cv=skf.split(X_train,y_train), verbose=1,)
 print(out.mean() ,out.std() ,out.mean() -out.std())
<save_to_csv>
cat_feat_to_encode = binary_vars + ordinal_vars + nominal_vars + time_vars
smoothing = 0.3
enc_x = np.zeros(X[cat_feat_to_encode].shape)
 for tr_idx, oof_idx in skf.split(X, y):
encoder = cat_encs.TargetEncoder(cols=cat_feat_to_encode, smoothing=smoothing)
 encoder.fit(X[cat_feat_to_encode].iloc[tr_idx], y[tr_idx])
 enc_x[oof_idx, :] = encoder.transform(X[cat_feat_to_encode].iloc[oof_idx], y[oof_idx])
 encoder.fit(X[cat_feat_to_encode], y)
 enc_xt = encoder.transform(Xt[cat_feat_to_encode] ).values
for idx, new_var in enumerate(cat_feat_to_encode):
new_var = new_var + '_enc'
X[new_var] = enc_x[:,idx]
Xt[new_var] = enc_xt[:, idx]
model.fit(X_train,y_train)
 pred_test = model.predict_proba(test)[:,1]
sub = pd.DataFrame({'ID_code':test.index,'target':pred_test})
 sub.to_csv('my_lgb_sub.csv',index=False)
 sub.head()<set_options>
X = X.astype(np.float32)
 Xt = Xt.astype(np.float32)
 cat_features = nominal_vars + ordinal_vars
X[cat_features] = X[cat_features].astype(np.int64)
 Xt[cat_features] = Xt[cat_features].astype(np.int64 )
%matplotlib inline<load_from_csv>
best_params = {'bagging_temperature': 0.8,
'depth': 5,
'iterations': 1000,
'l2_leaf_reg': 30,
'learning_rate': 0.05,
'random_strength': 0.8}
train_df = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df = pd.read_csv(".. /input/digit-recognizer/test.csv" )<count_values>
roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
cv_test_preds = np.zeros(len(Xt))
best_iteration = list()
for train_idx, test_idx in skf.split(X, y):
X_train, y_train = X.iloc[train_idx, :], y[train_idx]
X_test, y_test = X.iloc[test_idx, :], y[test_idx]
train = Pool(data=X_train,
label=y_train,
feature_names=list(X_train.columns),
cat_features=cat_features)
 val = Pool(data=X_test,
label=y_test,
feature_names=list(X_test.columns),
cat_features=cat_features)
 catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False)
 catb.fit(train,
verbose_eval=100,
early_stopping_rounds=50,
eval_set=val,
use_best_model=True,
plot=False)
 best_iteration.append(catb.best_iteration_)
 preds = catb.predict_proba(X_test)
 oof[test_idx] = preds[:,1]
Xt_pool = Pool(data=Xt[list(X_train.columns)],
feature_names=list(X_train.columns),
cat_features=cat_features)
 cv_test_preds += catb.predict_proba(Xt_pool)[:,1] / FOLDS
roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))
average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))
train_df['label'].value_counts(normalize=True)
<prepare_x_and_y>
oof = pd.DataFrame({'id':id_train, 'catboost_oof': oof})
 oof.to_csv("oof.csv", index=False)
 cv_submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 cv_submission.target = cv_test_preds
cv_submission.to_csv("./catboost_cv_submission.csv", index=False )
x_train = train_df.drop(labels = ["label"],axis = 1)
 y_train = train_df['label']
del train_df<categorify>
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof.catboost_oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof.catboost_oof))
y_train = to_categorical(y_train, num_classes=10)
 y_train[:2,:]<split>
catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False)
 train = Pool(data=X,
label=y,
feature_names=list(X_train.columns),
cat_features=cat_features)
 catb.fit(train,
verbose_eval=100,
plot=False)
 Xt_pool = Pool(data=Xt[list(X_train.columns)],
feature_names=list(X_train.columns),
cat_features=cat_features)
 submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 submission.target = catb.predict_proba(Xt_pool)[:,1]
submission.to_csv("./submission.csv", index=False )