Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<categorify>
__seed = 0
__n_folds = 5
__nrows = None
%matplotlib inline
plt.style.use('ggplot')
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 500)
 folds1 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed)
 folds2 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed+2)
 folds3 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed+4 )
THRESHOLD_P_VALUE = 0.01
THRESHOLD_STATISTIC = 0.3
diff_cols = []
for col in train.columns:
statistic, pvalue = ks_2samp(train[col].values, test[col].values)
 if pvalue <= THRESHOLD_P_VALUE and np.abs(statistic)> THRESHOLD_STATISTIC:
diff_cols.append(col)
 for col in diff_cols:
if col in train.columns:
train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True)
 train.shape<feature_engineering>
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv", index_col = "id", nrows = __nrows)
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv", index_col = "id", nrows = __nrows)
 train.head()
ntrain = len(train)
 ntest = len(test)
 tmp = pd.concat([train,test])
 weight =(( train != 0 ).sum() /len(train)).values
tmp_train = train[train!=0]
tmp_test = test[test!=0]
train["weight_count"] =(tmp_train*weight ).sum(axis=1)
 test["weight_count"] =(tmp_test*weight ).sum(axis=1)
 train["count_not0"] =(train != 0 ).sum(axis=1)
 test["count_not0"] =(test != 0 ).sum(axis=1)
 train["sum"] = train.sum(axis=1)
 test["sum"] = test.sum(axis=1)
 train["var"] = tmp_train.var(axis=1)
 test["var"] = tmp_test.var(axis=1)
 train["median"] = tmp_train.median(axis=1)
 test["median"] = tmp_test.median(axis=1)
 train["mean"] = tmp_train.mean(axis=1)
 test["mean"] = tmp_test.mean(axis=1)
 train["std"] = tmp_train.std(axis=1)
 test["std"] = tmp_test.std(axis=1)
 train["max"] = tmp_train.max(axis=1)
 test["max"] = tmp_test.max(axis=1)
 train["min"] = tmp_train.min(axis=1)
 test["min"] = tmp_test.min(axis=1)
 train["skew"] = tmp_train.skew(axis=1)
 test["skew"] = tmp_test.skew(axis=1)
 train["kurtosis"] = tmp_train.kurtosis(axis=1)
 test["kurtosis"] = tmp_test.kurtosis(axis=1)
 del(tmp_train)
 del(tmp_test)
 NUM_OF_COM = 100
transformer = random_projection.SparseRandomProjection(n_components = NUM_OF_COM)
 RP = transformer.fit_transform(tmp)
 rp = pd.DataFrame(RP)
 columns = ["RandomProjection{}".format(i)for i in range(NUM_OF_COM)]
rp.columns = columns
rp_train = rp[:ntrain]
rp_test = rp[ntrain:]
rp_test.index = test.index
train = pd.concat([train,rp_train],axis=1)
 test = pd.concat([test,rp_test],axis=1)
 del(rp_train)
 del(rp_test)
 train.shape<compute_train_metric>
def fit_describe_infos(train, test, __featToExcl = [], target_for_vcramer = None):

 stats = []
__featToAnalyze = [v for v in list(train.columns)if v not in __featToExcl]
for col in tqdm_notebook(__featToAnalyze):
dtrain = dict(train[col].value_counts())
 dtest = dict(test[col].value_counts())
 set_train_not_in_test = set(dtest.keys())- set(dtrain.keys())
 set_test_not_in_train = set(dtrain.keys())- set(dtest.keys())
 dict_train_not_in_test = {key:value for key, value in dtest.items() if key in set_train_not_in_test}
dict_test_not_in_train = {key:value for key, value in dtrain.items() if key in set_test_not_in_train}
nb_moda_test, nb_var_test = len(dtest), pd.Series(dtest ).sum()
nb_moda_abs, nb_var_abs = len(dict_train_not_in_test), pd.Series(dict_train_not_in_test ).sum()
nb_moda_train, nb_var_train = len(dtrain), pd.Series(dtrain ).sum()
nb_moda_abs_2, nb_var_abs_2 = len(dict_test_not_in_train), pd.Series(dict_test_not_in_train ).sum()
if not target_for_vcramer is None:
vc = coef_vcramer(pd.crosstab(train[target_for_vcramer], train[col].fillna(-1)))
 else:
vc = 0
stats.append(( col, round(vc, 3), train[col].nunique()
, str(nb_moda_abs)+ '(' + str(round(100 * nb_moda_abs / nb_moda_test, 1)) +'%)'
, str(nb_moda_abs_2)+'(' + str(round(100 * nb_moda_abs_2 / nb_moda_train, 1)) +'%)'
, str(train[col].isnull().sum())+'(' + str(round(100 * train[col].isnull().sum() / train.shape[0], 1)) +'%)'
, str(test[col].isnull().sum())+'(' + str(round(100 * test[col].isnull().sum() / test.shape[0], 1)) +'%)'
, str(round(100 * train[col].value_counts(normalize = True, dropna = False ).values[0], 1))
, train[col].dtype))
df_stats = pd.DataFrame(stats, columns=['Feature', "Target Cramer's V"
, 'Unique values(train)', "Unique values in test not in train(and %)"
, "Unique values in train not in test(and %)"
, 'NaN in train(and %)', 'NaN in test(and %)', '% in the biggest cat.( train)'
, 'dtype'])
 if target_for_vcramer is None:
df_stats.drop("Target Cramer's V", axis=1, inplace=True)
 return df_stats, dict_train_not_in_test, dict_test_not_in_train
NUM_FOLDS = 5
def rmsle_cv(model):
kf = KFold(NUM_FOLDS, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([ model.predict(X)for model in self.models_ ])
 return np.mean(predictions, axis=1)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.055, colsample_bylevel =0.5,
gamma=1.5, learning_rate=0.02, max_depth=32,
objective='reg:linear',booster='gbtree',
min_child_weight=57, n_estimators=1000, reg_alpha=0,
reg_lambda = 0,eval_metric = 'rmse', subsample=0.7,
silent=1, n_jobs = -1, early_stopping_rounds = 14,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=144,
learning_rate=0.005, n_estimators=720, max_depth=13,
metric='rmse',is_training_metric=True,
max_bin = 55, bagging_fraction = 0.8,verbose=-1,
bagging_freq = 5, feature_fraction = 0.9)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models = AveragingModels(models =(model_xgb, model_lgb))
score = rmsle_cv(averaged_models)
 print("averaged score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<save_to_csv>
lr_params = {'penalty': 'l2', 'solver': 'lbfgs', 'C': 0.123456789, 'max_iter':500}
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
[f for f in train.columns if not f in ["target", "bin_0"]])
 pipe = make_pipeline(make_column_transformer(ohe1), LogisticRegression(**lr_params))
averaged_models.fit(train.values, y_train)
 pred = np.expm1(averaged_models.predict(test.values))
ensemble = pred
sub = pd.DataFrame()
sub['ID'] = test_ID
sub['target'] = ensemble
sub.to_csv('submission.csv',index=False)
<import_modules>
_, best_score1 = cross_val_and_print(pipe)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3 )
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
from sklearn.model_selection import KFold<load_from_csv>
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
[f for f in train.columns if not f in ["target"]])
 pipe = make_pipeline(make_column_transformer(ohe1), LogisticRegression(**lr_params))
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )<load_from_csv>
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2 )
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )<load_from_csv>
for df in [train, test]:
df["bin_0_bin_3"] = df["bin_3"].astype(str)+ df["bin_0"].astype(str )
report = pd.read_csv('.. /input/feature-report/feature_report.csv', index_col='feature' )<filter>
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
[f for f in train.columns if not f in ["target", "bin_0", "bin_3"]])
 pipe = make_pipeline(make_column_transformer(ohe1), LogisticRegression(**lr_params))
good_features = report.loc[report['rmse'] <= 0.7955].index
rmses = report.loc[report['rmse'] <= 0.7955, 'rmse'].values
good_features<load_from_csv>
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )<load_from_csv>
def transf_ordinal_features(serie):
dtransf = {"ord_1":{'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
, "ord_2":{'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
, "nom_0":{"Blue":1, "Green":2, "Red":3}
, "nom_1":{"Circle":1, "Trapezoid":2, "Star":3, "Polygon":4, "Square":5, "Triangle":6}
, "nom_2":{"Dog":1, "Lion":2, "Snake":3, "Axolotl":4, "Cat":5, "Hamster":6}
, "nom_3":{"Finland":1, "Russia":2, "China":3, "Costa Rica":4, "Canada":5, "India":6}
, "nom_4":{"Bassoon":1, "Piano":2, "Oboe":3, "Theremin":4}
, "bin_0_bin_3_bis":{"T0":0, "F1":2, "F0":1}
}
if serie.name == "ord_0":
new_serie = serie - 1
elif serie.name == "ord_5":
lm = serie.unique()
new_serie = serie.map({l:i for i, l in enumerate(list(np.sort(lm)))})
 elif serie.name in ["ord_3", "ord_4"]:
new_serie = serie.str.lower().map({l:i for i, l in enumerate(list(ascii_lowercase)) })
 else:
new_serie = serie.map(dtransf[serie.name])
 return new_serie
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )<import_modules>
class MyFeaturesEngineering(BaseEstimator, TransformerMixin):
def __init__(self, list_ordinal_features =[]):
self.list_ordinal_features = list_ordinal_features
def fit(self, x, y=None):
return self
def transform(self, x, y=None):
df = x.copy()
for v in self.list_ordinal_features:
df[v] = transf_ordinal_features(df[v])
 return df
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
import lightgbm as lgb<feature_engineering>
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"]
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
for feat in ordinal_features:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
list(set(ohe1_feats+ordinal_features)-{feat}))
MyFeE = MyFeaturesEngineering(list_ordinal_features = [feat])
 StdScalE =(StandardScaler(copy=False), [feat])
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat)
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1 )<prepare_x_and_y>
ordinal_features = ["ord_0"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4", "nom_5"
, "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"] + ["ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0] )<import_modules>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
from skopt import BayesSearchCV
from sklearn.model_selection import StratifiedKFold, KFold<find_best_params>
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best MSE: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
clf_name = bayes_cv_tuner.estimator.__class__.__name__
all_models.to_csv(clf_name+"_cv_results.csv" )<compute_test_metric>
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5<choose_model_class>
class MyBinsEncoder(BaseEstimator, TransformerMixin):
def __init__(self, nbins=100, nmin=20):
self.nbins = nbins
self.nmin = nmin
def fit(self, x, y=None):
temp = pd.concat([x, y], axis=1)
 averages = temp.groupby(by=x.name)[y.name].mean()
means = dict(zip(averages.index.values, averages.values))
bins = np.linspace(averages.min() , averages.max() , self.nbins)
 self.map_ = dict(zip(averages.index.values, np.digitize(averages.values, bins=bins)))
 count = temp.groupby(by=x.name)[y.name].count()
nobs = dict(zip(averages.index.values, count))
for key, value in nobs.items() :
if value > self.nmin:
self.map_[key] = key
return self
def transform(self, x, y=None):
temp = x.map(self.map_)
 temp.fillna(random.choice(list(self.map_.values())) , inplace=True)
 temp = temp.astype(str)
 return temp
essai = MyBinsEncoder(nbins=4, nmin=100000)
 essai.fit(train["day"], train["target"])
 print(essai.map_)
 av = essai.transform(train["day"])
 print(av.value_counts(dropna=False))
print(av.nunique() )
%%time
bayes_cv_tuner = BayesSearchCV(
 estimator = lgb.LGBMRegressor(objective='regression', boosting_type='gbdt', subsample=0.6143),
search_spaces = {
'learning_rate':(0.01, 1.0, 'log-uniform'),
'num_leaves':(10, 100),
'max_depth':(0, 50),
'min_child_samples':(0, 50),
'max_bin':(100, 1000),
'subsample_freq':(0, 10),
'min_child_weight':(0, 10),
'reg_lambda':(1e-9, 1000, 'log-uniform'),
'reg_alpha':(1e-9, 1.0, 'log-uniform'),
'scale_pos_weight':(1e-6, 500, 'log-uniform'),
'n_estimators':(50, 150),
},
scoring = 'neg_mean_squared_log_error',
cv = KFold(
 n_splits=5,
shuffle=True,
random_state=42 
),
n_jobs = 1,
n_iter = 100,
verbose = 0,
refit = True,
random_state = 42 
)
 result = bayes_cv_tuner.fit(data[features], target, callback=status_print )<predict_on_test>
class MyFeaturesEngineering(BaseEstimator, TransformerMixin):
def __init__(self, list_ordinal_features =[], feat_to_bins_encode = {}):
self.list_ordinal_features = list_ordinal_features
self.feat_to_bins_encode = feat_to_bins_encode
self.BinsEncoder={}
def fit(self, x, y=None):
for feat, value in self.feat_to_bins_encode.items() :
self.BinsEncoder[feat] = MyBinsEncoder(nbins=value[0], nmin=value[1])
 self.BinsEncoder[feat].fit(x[feat], y)
 return self
def transform(self, x, y=None):
df = x.copy()
for v in self.feat_to_bins_encode.keys() :
df[v] = self.BinsEncoder[v].transform(df[v])
 for v in self.list_ordinal_features:
df[v] = transf_ordinal_features(df[v])
 return df
pred = bayes_cv_tuner.predict(test[features] )<save_to_csv>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_5"
for nbins in [[20, 35]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )
test['target'] = np.expm1(pred)
 test[['ID', 'target']].to_csv('my_submission.csv', index=False, float_format='%.2f' )<save_to_csv>
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )
test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('submission.csv', index=False, float_format='%.2f' )<import_modules>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_6"
for nbins in [[20, 5]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins, "nom_5":[20, 35]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )
import numpy as np
import pandas as p
from matplotlib import pyplot as plt
import seaborn as sns
from math import sqrt
from sklearn import metrics<load_from_csv>
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )
df_base0 = p.read_csv('.. /input/1a2a3a/final_submission/1a.csv',names=["ID","target"], skiprows=[0],header=None)
 df_base1 = p.read_csv('.. /input/1a2a3a/final_submission/2a.csv',names=["ID","target"], skiprows=[0],header=None)
 df_base2 = p.read_csv('.. /input/1a2a3a/final_submission/3a.csv',names=["ID","target"], skiprows=[0],header=None )<merge>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_8"
for nbins in [[15, 3]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins, "nom_5":[20, 35]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )
df_base = p.merge(df_base0,df_base1,how='inner',on='ID')
 df_base = p.merge(df_base,df_base2,how='inner',on='ID' )<compute_test_metric>
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<feature_engineering>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_8", "day_bis", "month"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop="first")
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat )
df_base['target'] = df_base.iloc[:,1:].mean(axis=1 )<save_to_csv>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_8", "day_bis", "month"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat )
df_base[['ID','target']].to_csv("latest.csv",index=False )<import_modules>
ohe1_feats = ["bin_0_bin_3", "bin_1", "nom_5", "nom_7", "nom_9", "month"]
ohe2_feats = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_6', "day_bis"]
ohe3_feats = ['bin_2', 'bin_4', 'nom_8']
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 lval_to_drop = [train[v].value_counts().index[0] for v in ohe2_feats]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , ohe2_feats)
 ohe3 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first'), ohe3_feats)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
import tensorflow as tf
import os<load_from_csv>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_6', "day_bis"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat)
sub_df = pd.read_csv(data_path+'sample_submission.csv')
 print("test data 수:",len(sub_df))
sub_df.head()<define_variables>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['nom_1', 'nom_3', "day_bis"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat )
train_file_path = "/kaggle/input/quickdraw-doodle-recognition/train_raw/"<define_variables>
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['bin_2', 'bin_4', 'nom_8']:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first')
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat)
train_csvs= os.listdir(train_file_path)
 print("train_raw 폴더 내 파일 수:", len(train_csvs))
print(train_csvs[:5])
 file_size = 0
label_names = []
for csv_file in train_csvs:
file_size += os.path.getsize(train_file_path + csv_file)
 label_names.append(csv_file.replace('.csv',''))
print("파일 크기 : ", file_size//(1024*1024*1024),"GB")
 label_names = sorted(label_names,key=lambda x : str.lower(x+'.csv'))<define_variables>
ohe1_feats = ["bin_0_bin_3", "bin_1", 'bin_2', 'bin_4', 'nom_0', 'nom_2', 'nom_4', "nom_5", 'nom_6'
, "nom_7", "nom_9", "month"]
ohe2_feats = ['nom_1', 'nom_3', "day_bis"]
ohe3_feats = ['nom_8']
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 lval_to_drop = [train[v].value_counts().index[0] for v in ohe2_feats]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , ohe2_feats)
 ohe3 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first'), ohe3_feats)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )
<import_modules><EOS> 
lr_params = {'penalty': 'l2', 'solver': 'lbfgs', 'C': 0.123456789, 'max_iter':500}
pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
pred = pipe.fit(train.drop(["target"], axis = 1), train["target"] ).predict_proba(test)[:, 1]
pd.DataFrame({"id": test.index, "target": pred} ).to_csv("submission.csv", index=False )