Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<load_from_csv>
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
from sklearn.model_selection import KFold
sub = pd.read_csv(test )<categorify>
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )
data_sub = bert_encode(sub, tokenizer )<prepare_output>
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )
sub_val = clf([data_sub['input_word_ids'],
data_sub['input_mask'],
data_sub['input_type_ids']], training=False)
<define_variables>
report = pd.read_csv('.. /input/feature-report/feature_report.csv', index_col='feature' )
target = [1 if i>0 else 0 for i in sub_val]<save_to_csv>
good_features = report.loc[report['rmse'] <= 0.7955].index
rmses = report.loc[report['rmse'] <= 0.7955, 'rmse'].values
good_features
my_submission = pd.DataFrame({'id': sub.id, 'Target': target})
 my_submission.to_csv('submission.csv', index=False )<load_from_csv>
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
<drop_column>
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )
colsToRemove = []
for col in train.columns:
if train[col].std() == 0:
colsToRemove.append(col)
 train.drop(colsToRemove, axis=1, inplace=True)
 test.drop(colsToRemove, axis=1, inplace=True)
 colsToRemove = []
columns = train.columns
for i in range(len(columns)-1):
v = train[columns[i]].values
for j in range(i+1,len(columns)) :
if np.array_equal(v,train[columns[j]].values):
colsToRemove.append(columns[j])
 train.drop(colsToRemove, axis=1, inplace=True)
 test.drop(colsToRemove, axis=1, inplace=True)
 train.shape<drop_column>
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
import lightgbm as lgb
target=train.TARGET.values
train.drop(['ID','TARGET'],axis=1,inplace = True)
 ids=test.ID.values
test.drop(['ID'],axis=1,inplace = True)
 print(train.shape,'
',test.shape)
<normalization>
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1 )
slct = SelectFpr(alpha = 0.000001)
 trainFeatures = slct.fit_transform(train, target)
 print(trainFeatures.shape )<features_selection>
features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0] )
colsToRetain = slct.get_support(indices = True)
<split>
from skopt import BayesSearchCV
from sklearn.model_selection import StratifiedKFold, KFold
clf = xgb.XGBClassifier(missing=np.nan, max_depth=5, n_estimators=550, learning_rate=0.03, nthread=4, subsample=0.95, colsample_bytree=0.85, seed=4242)
 X_train, X_test, y_train, y_test = train_test_split(trainFeatures, target, test_size=0.3 )<split>
def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best MSE: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
clf_name = bayes_cv_tuner.estimator.__class__.__name__
all_models.to_csv(clf_name+"_cv_results.csv" )
X_fit, X_eval, y_fit, y_eval= train_test_split(trainFeatures, target, test_size=0.3 )<save_to_csv>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
clf = xgb.XGBClassifier(missing=np.nan, max_depth=5, n_estimators=350, learning_rate=0.03, nthread=4,
subsample=0.95, colsample_bytree=0.85, seed=4242)
 clf.fit(trainFeatures, target, early_stopping_rounds=365, eval_metric="auc", eval_set=[(X_eval, y_eval)])
<compute_test_metric>
%%time
bayes_cv_tuner = BayesSearchCV(
 estimator = lgb.LGBMRegressor(objective='regression', boosting_type='gbdt', subsample=0.6143),
search_spaces = {
'learning_rate':(0.01, 1.0, 'log-uniform'),
'num_leaves':(10, 100),
'max_depth':(0, 50),
'min_child_samples':(0, 50),
'max_bin':(100, 1000),
'subsample_freq':(0, 10),
'min_child_weight':(0, 10),
'reg_lambda':(1e-9, 1000, 'log-uniform'),
'reg_alpha':(1e-9, 1.0, 'log-uniform'),
'scale_pos_weight':(1e-6, 500, 'log-uniform'),
'n_estimators':(50, 150),
},
scoring = 'neg_mean_squared_log_error',
cv = KFold(
 n_splits=5,
shuffle=True,
random_state=42 
),
n_jobs = 1,
n_iter = 100,
verbose = 0,
refit = True,
random_state = 42 
)
 result = bayes_cv_tuner.fit(data[features], target, callback=status_print )
print('Overall AUC:', roc_auc_score(target, clf.predict_proba(trainFeatures)[:,1]))<save_to_csv>
pred = bayes_cv_tuner.predict(test[features] )
y_pred= clf.predict_proba(testFeatures)[:,1]
submission = pd.DataFrame({"ID":ids, "TARGET":y_pred})
 submission.to_csv("submission.csv", index=False)
 print('done !' )<set_options>
test['target'] = np.expm1(pred)
 test[['ID', 'target']].to_csv('my_submission.csv', index=False, float_format='%.2f' )
<load_from_csv><EOS> 
test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('submission.csv', index=False, float_format='%.2f' )