Kaggle competition: Santander Value Prediction Challenge
def modelfit(alg, dtrain, predictors,useTrainCV=True, cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = alg.get_xgb_params()
xgtrain = xgb.DMatrix(dtrain,label = predictors)
 cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params() ['n_estimators'], nfold=cv_folds,
metrics='auc', early_stopping_rounds=early_stopping_rounds, show_progress=False)
 alg.set_params(n_estimators=cvresult.shape[0])
 alg.fit(predictors, dtrain['Disbursed'],eval_metric='auc')
 dtrain_predictions = alg.predict(predictors)
 dtrain_predprob = alg.predict_proba(predictors)[:,1]
print("
Model Report")
 print("Accuracy : %.4g" % metrics.accuracy_score(dtrain['Disbursed'].values, dtrain_predictions))
print("AUC Score(Train): %f" % metrics.roc_auc_score(dtrain['Disbursed'], dtrain_predprob))
feat_imp = pd.Series(alg.booster().get_fscore() ).sort_values(ascending=False)
 feat_imp.plot(kind='bar', title='Feature Importances')
 plt.ylabel('Feature Importance Score')
 print("Done" )<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values
xgb1 = xgb.XGBClassifier(
 learning_rate =0.1,
n_estimators=1000,
max_depth=5,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb1, X_train, y_train )<set_options>
test["target"] = train["target"].mean()
sns.set_style('whitegrid')
 %matplotlib inline 
<load_from_csv>
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535',
'fc99f9426', '91f701ba2', '0572565c2', '190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'
]
santander_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 santander_df.head()<drop_column>
def _get_leak(df, cols, lag=0):
d1 = df[cols[:-lag-2]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2 = df[cols[lag+2:]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = df[cols[lag]]
d3 = d2[~d2.duplicated(['key'], keep=False)]
return d1.merge(d3, how='left', on='key' ).pred.fillna(0 )
columns_to_be_removed = []
cunt_constant = 0
for col in santander_df.columns:
if santander_df[col].std() == 0:
columns_to_be_removed.append(col)
 cunt_constant = cunt_constant + 1
santander_df.drop(columns_to_be_removed, axis=1, inplace=True)
 test_df.drop(columns_to_be_removed, axis=1, inplace=True)
 columns_to_be_removed = []
cunt_duplicates = 0
for col_i in santander_df.columns:
for col_j in santander_df.columns:
if(col_i == col_j): continue
if(( santander_df[col_i] == santander_df[col_j] ).all()):
columns_to_be_removed.append(col_i)
 cunt_duplicates = cunt_duplicates + 1
break
santander_df.drop(columns_to_be_removed, axis=1, inplace=True)
 test_df.drop(columns_to_be_removed, axis=1, inplace=True )<count_unique_values>
def compiled_leak_result() :
max_nlags = len(cols)- 2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = _get_leak(train_leak, cols, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result
cunt_nunique = santander_df.apply(lambda col:col.nunique())
 cunt_nunique.drop("ID", inplace=True)
 cunt_nunique[cunt_nunique <= 100].hist(bins=100, figsize=(15, 5))<filter>
result = pd.DataFrame.from_dict(result, orient='columns')
 result.T
most_common_num = santander_df['var3'].value_counts().index[0]
santander_df["var3"][santander_df["var3"] == -999999] = most_common_num<prepare_x_and_y>
result.to_csv('train_leaky_stat.csv', index=False )
X_train = santander_df.drop(["ID","TARGET"],axis=1)
 Y_train = santander_df["TARGET"]
X_test = test_df.drop("ID",axis=1 ).copy()<find_best_model_class>
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict_proba(X_test)[:,1]
logreg.score(X_train, Y_train )<train_model>
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict_proba(X_test)[:,1]
random_forest.score(X_train, Y_train )<predict_on_test>
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()
params = {"objective": "binary:logistic"}
T_train_xgb = xgb.DMatrix(X_train, Y_train)
 X_test_xgb = xgb.DMatrix(X_test)
 gbm = xgb.train(params, T_train_xgb, 20)
 Y_pred = gbm.predict(X_test_xgb )<prepare_output>
train_res = train_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 train_res.to_csv('train_leak.csv', index=False )
coeff_df = DataFrame(X_train.columns)
 coeff_df.columns = ['Features']
coeff_df["Coefficient Estimate"] =(pd.Series(logreg.coef_[0])) ** 2
coeff_df.head()<save_to_csv>
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = _get_leak(test_leak, cols, i)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result
submission = pd.DataFrame()
submission["ID"] = test_df["ID"]
submission["TARGET"] = Y_pred
submission.to_csv('santander.csv', index=False )<prepare_x_and_y>
test_leak, test_result = compiled_leak_result_test(max_nlags=38 )
y_train = df_train['TARGET'].values
X_train = df_train.drop(['ID','TARGET'], axis=1 ).values
id_test = df_test['ID']
X_test = df_test.drop(['ID'], axis=1 ).values
len_train = len(X_train)
 len_test = len(X_test)
 clf = xgb.XGBClassifier(missing=np.nan, max_depth=5, n_estimators=350, learning_rate=0.03, nthread=4, subsample=0.95, colsample_bytree=0.85, seed=4242)
 X_fit, X_eval, y_fit, y_eval= train_test_split(X_train, y_train, test_size=0.3)
 clf.fit(X_train, y_train, early_stopping_rounds=20, eval_metric="auc", eval_set=[(X_eval, y_eval)])
 print('Overall AUC:', roc_auc_score(y_train, clf.predict_proba(X_train)[:,1]))
y_pred= clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame({"ID":id_test, "TARGET":y_pred})
 submission.to_csv("submission_2016_03_15.csv", index=False)
 print('Completed!' )<set_options>
test_result = pd.DataFrame.from_dict(test_result, orient='columns')
 test_result.T
logging.basicConfig(level=logging.INFO )<load_from_url>
test_result.to_csv('test_leaky_stat.csv', index=False )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<choose_model_class>
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+['compiled_leak']].head()
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<load_from_csv>
test_res = test_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 test_res.to_csv('test_leak.csv', index=False )
train = pd.read_csv('train.tsv', sep='\t')
 test = pd.read_csv('test.tsv', sep='\t' )<choose_model_class>
test_leak.loc[test_leak["compiled_leak"]==0, "compiled_leak"] = test_leak.loc[test_leak["compiled_leak"]==0, "nonzero_mean"]
<categorify><EOS> 
sub = test[["ID"]]
sub["target"] = test_leak["compiled_leak"]
sub.to_csv(f"baseline_sub_lag_{best_lag}.csv", index=False)
 print(f"baseline_sub_lag_{best_lag}.csv saved" )