Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<load_from_csv>
ver = 'logreg'
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', nrows=2000000)
 train.info()<feature_engineering>
import numpy as np
import pandas as pd
train = train[train['weight']!=0]
train['action'] = train['resp'].apply(lambda x:x>0 ).astype(int )<define_variables>
warnings.simplefilter(action = 'ignore' )
features = [col for col in list(train.columns)if 'feature' in col]<split>
from sklearn.metrics import roc_auc_score, log_loss, accuracy_score, confusion_matrix
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
X = train[features]
y = train['action']
train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.2 )<feature_engineering>
from scipy.stats import mannwhitneyu
missing_values = pd.DataFrame()
missing_values['feature'] = features
missing_values['num_missing'] = [train_x[i].isna().sum() for i in features]
missing_values.T<categorify>
train = pd.read_csv('.. /input/train.csv', index_col = 'id')
 train.head()
train_median = train_x.median()
train_x = train_x.fillna(train_median)
 valid_x = valid_x.fillna(train_median )<normalization>
target = train['target']
train.drop('target', axis = 1, inplace = True)
 target.value_counts()
scaler = StandardScaler()
scaler.fit(train_x)
 train_x_norm = scaler.transform(train_x)
 pca = PCA()
comp = pca.fit(train_x_norm)
 plt.plot(np.cumsum(comp.explained_variance_ratio_))
plt.grid()
plt.xlabel('Number of Principal Components')
 plt.ylabel('Explained Variance')
 sns.despine() ;<normalization>
test = pd.read_csv('.. /input/test.csv', index_col = 'id')
 test.head()
pca = PCA(n_components=50 ).fit(train_x_norm)
 train_x_transform = pca.transform(train_x_norm )<normalization>
def save_submit(test_, clfs_, filename):
subm = pd.DataFrame(np.zeros(test_.shape[0]), index = test_.index, columns = ['target'])
 for clf in clfs_:
subm['target'] += clf.predict_proba(test_)[:, 1]
subm['target'] /= len(clfs_)
 subm = subm.reset_index()
subm.columns = ['id', 'target']
subm.to_csv(filename, index = False )
valid_x_transform = pca.transform(scaler.transform(valid_x))<prepare_x_and_y>
scores = pd.DataFrame(columns = ['auc', 'acc', 'loss', 'tn', 'fn', 'fp', 'tp'] )
dtrain = xgb.DMatrix(train_x_transform, label=train_y)
 dvalid = xgb.DMatrix(valid_x_transform, label=valid_y )<find_best_params>
mw = pd.DataFrame(index = ['stat', 'p'])
 for c in train.columns:
mw[c] = mannwhitneyu(train[c], test[c])
 mw = mw.T
mw[mw['p'] <.01].shape
def objective(trial):
params = {
'n_estimators': trial.suggest_int('n_estimators', 200, 600),
'max_depth': trial.suggest_int('max_depth', 10, 25),
'learning_rate': trial.suggest_uniform('learning_rate', 0.01, 0.1),
'subsample': trial.suggest_uniform('subsample', 0.50, 1),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.50, 1),
'gamma': trial.suggest_int('gamma', 0, 10),
'tree_method': 'gpu_hist',
'objective': 'binary:logistic'
}
bst = xgb.train(params, dtrain)
 preds = bst.predict(dvalid)
 pred_labels = np.rint(preds)
 accuracy = sklearn.metrics.accuracy_score(valid_y, pred_labels)
 return accuracy<find_best_params>
bad_features = list(mw[mw['p'] <.01].index)
 len(bad_features )
if __name__ == "__main__":
study = optuna.create_study(direction="maximize")
 study.optimize(objective, n_trials=25, timeout=600)
 print("Number of finished trials: ", len(study.trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))<find_best_params>
def logreg_cross_validation(train_, target_, params,
num_folds = 5, repeats = 20, rs = 0):
print(params)
 clfs = []
folds = RepeatedStratifiedKFold(n_splits = num_folds, n_repeats = repeats, random_state = rs)
 valid_pred = pd.DataFrame(index = train_.index)
 for n_fold,(train_idx, valid_idx)in enumerate(folds.split(target_, target_)) :
print('--- Fold {} started at {}'.format(n_fold, time.ctime()))
train_x, train_y = train_.iloc[train_idx], target_.iloc[train_idx]
valid_x, valid_y = train_.iloc[valid_idx], target_.iloc[valid_idx]
clf = LogisticRegression(**params)
 clf.fit(train_x, train_y)
 clfs.append(clf)
 predict = clf.predict_proba(valid_x)[:, 1]
tn, fp, fn, tp = confusion_matrix(valid_y,(predict >=.5)* 1 ).ravel()
auc = roc_auc_score(valid_y, predict)
 acc = accuracy_score(valid_y,(predict >=.5)* 1)
 loss = log_loss(valid_y, predict)
 print('TN =', tn, 'FN =', fn, 'FP =', fp, 'TP =', tp)
 print('AUC = ', auc, 'Loss =', loss, 'Acc =', acc)
 valid_pred[n_fold] = pd.Series(predict, index = valid_x.index)
 del train_x, train_y, valid_x, valid_y, predict
gc.collect()
return clfs, valid_pred
best_params = trial.params
best_params['tree_method'] = 'gpu_hist'
best_params['objective'] = 'binary:logistic' 
<choose_model_class>
params = {}
params['random_state'] = 0
params['n_jobs'] = -1
params['C'] =.2
params['penalty'] = 'l1'
params['class_weight'] = 'balanced'
params['solver'] = 'saga'
optimal_clf = xgb.XGBClassifier(**best_params )<train_model>
clfs, pred = logreg_cross_validation(train.drop(bad_features, axis = 1), target, params )
optimal_clf.fit(train_x_transform, train_y )<categorify>
pred_mean = pred.mean(axis = 1 )
def fillna_npwhere(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array<define_variables>
scores = scores.T
tn, fp, fn, tp = confusion_matrix(target,(pred_mean >=.5)* 1 ).ravel()
scores['logreg'] = [
roc_auc_score(target, pred_mean),
accuracy_score(target,(pred_mean >=.5)* 1),
log_loss(target, pred_mean),
tn, fn, fp, tp]
scores = scores.T
scores
env = janestreet.make_env()
iter_test = env.iter_test()<predict_on_test>
score_auc = scores.loc['logreg', 'auc']
score_acc = scores.loc['logreg', 'acc']
score_loss = scores.loc['logreg', 'loss']
print(score_auc, score_acc, score_loss )
<set_options><EOS> 
loc_ver = 'v1'
filename = 'subm_{}_{}_{:.4f}_{:.4f}_{:.4f}_{}.csv'.format(ver, loc_ver, score_auc, score_acc, score_loss,
datetime.now().strftime('%Y-%m-%d'))
print(filename)
 save_submit(test.drop(bad_features, axis = 1), clfs, filename )