Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<install_modules>
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 train.sort_index(inplace=True)
 train_y = train['target']; test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True); test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = train.columns.tolist() ; smoothing=0.20
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, train_y)
 train = oof.sort_index()
test = ce_target_encoder.transform(test )
%%bash
cd LightGBM
rm -r build
mkdir build
cd build
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/..
make -j$(nproc )<install_modules>
glm = linear_model.LogisticRegression(random_state=1, solver='lbfgs', max_iter=2020, fit_intercept=True, penalty='none', verbose=0); glm.fit(train, train_y )
!cd LightGBM/python-package/;python3 setup.py install --precompile<set_options>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC, LogisticRegression
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
pd.set_option('display.max_columns', 200 )<load_from_csv>
n_folds = 5
def auc_score(model):
kf = KFold(n_folds, shuffle= True ).get_n_splits(train.values)
 auc_score = cross_val_score(model, train.values, train_y, scoring = "roc_auc", cv = kf)
 return auc_score
warnings.filterwarnings('ignore')
 def get_logger() :
FORMAT = '[%(levelname)s]%(asctime)s:%(name)s:%(message)s'
logging.basicConfig(format=FORMAT)
 logger = logging.getLogger('main')
 logger.setLevel(logging.DEBUG)
 return logger
logger = get_logger()
logger.info('Input data')
 train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 logger.info('Features engineering')
 idx = [c for c in train_df.columns if c not in ['ID_code', 'target']]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 logger.info('Prepare the model')
 features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']<set_options>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
!nvidia-smi<init_hyperparams>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
param = {
'num_leaves': 7,
'max_bin': 119,
'min_data_in_leaf': 6,
'learning_rate': 0.03,
'min_sum_hessian_in_leaf': 0.00245,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 4.972,
'lambda_l2': 2.276,
'min_gain_to_split': 0.65,
'max_depth': 14,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
'device': 'gpu',
'gpu_platform_id': 0,
'gpu_device_id': 0
}<split>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
%%time
nfold = 5
target = 'target'
predictors = train_df.columns.values.tolist() [2:]
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param, xg_train, 5000, valid_sets = [xg_valid], verbose_eval=500, early_stopping_rounds = 250)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[predictors], num_iteration=clf.best_iteration)/ nfold
i = i + 1
print("
CV AUC: {:<0.2f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )<prepare_output>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = predictions
sub_df[:10]<save_to_csv>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
sub_df.to_csv("lightgbm_gpu.csv", index=False )<set_options>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
warnings.filterwarnings("ignore")
 init_notebook_mode(connected=True)
<load_from_csv>
score = auc_score(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = auc_score(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = auc_score(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = auc_score(glm)
 print("Logistic Regression: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
".format(score.mean() , score.std()))
".format(score.mean() , score.std()))
".format(score.mean() , score.std()))
%%time
sctp_train = pd.read_csv('.. /input/train.csv')
 sctp_test = pd.read_csv('.. /input/test.csv')
 print("=="*45)
 print("The training dataset has {0} rows and {1} columns".format(sctp_train.shape[0], sctp_train.shape[1]))
print("The testing dataset has {0} rows and {1} columns".format(sctp_test.shape[0], sctp_test.shape[1]))
print("=="*45 )<count_missing_values>
class average_stacking(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self,models):
self.models = models
def fit(self, x,y):
self.model_clones = [clone(x)for x in self.models]
for model in self.model_clones:
model.fit(x,y)
 return self
def predict(self, x):
preds = np.column_stack([
model.predict(x)for model in self.model_clones
])
 return np.mean(preds, axis = 1 )
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Your selected dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns
missing_values_table(sctp_train )<filter>
averaged_models = average_stacking(models =(ENet, glm,model_lgb, lasso))
score = auc_score(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
corr_df_sel = corr_df.ix[(corr_df['corr_values'] < -0.05)|(corr_df['corr_values']>0.05)]
corr_df_sel<split>
averaged_models.fit(train.values, train_y)
 avg_pred = averaged_models.predict(test )
<train_model><EOS> 
pd.DataFrame({'id': test_id, 'target': avg_pred} ).to_csv('submission.csv', index=False )