Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<data_type_conversions>
import os, gc
import numpy as np
from numba import njit
import datatable as dtable
import pandas as pd
import janestreet
import xgboost as xgb
from hyperopt import hp, fmin, tpe, Trials
from hyperopt.pyll.base import scope
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import GroupKFold
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from tqdm.notebook import tqdm
from joblib import dump, load
fig, ax = plt.subplots(1, 10, figsize=(20, 10))
ax = ax.ravel()
img = x_val[0].reshape(1,x_val[0].shape[0],x_val[0].shape[1], x_val[0].shape[2])
 ax[0].imshow(img[0].astype('uint8'))
ax[1].imshow(next(ImageDataGenerator().flow(img)) [0].astype('uint8'))
ax[2].imshow(next(ImageDataGenerator(horizontal_flip=True, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[3].imshow(next(ImageDataGenerator(vertical_flip=True,fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[4].imshow(next(ImageDataGenerator(rotation_range=360, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[5].imshow(next(ImageDataGenerator(zoom_range=(0.65,1), fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[6].imshow(next(ImageDataGenerator(height_shift_range=0.15, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[7].imshow(next(ImageDataGenerator(width_shift_range=0.15, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[8].imshow(next(ImageDataGenerator(brightness_range=(0.5, 2), fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[9].imshow(next(ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
zoom_range=0.3,
brightness_range=(0.5, 2),
fill_mode='constant',cval=0 ).flow(img)) [0].astype('uint8')) 
<load_pretrained>
def weighted_average(a):
w = []
n = len(a)
 for j in range(1, n + 1):
j = 2 if j == 1 else j
w.append(1 /(2**(n + 1 - j)))
 return np.average(a, weights = w )
efficientnetb3 = EfficientNetB3(
 weights=None,
input_shape=(HEIGHT,WIDTH,3),
include_top=False 
)
 efficientnetb3.load_weights(".. /input/efficientnet-keras-weights-b0b5/efficientnet-b3_imagenet_1000_notop.h5" )<choose_model_class>
%%time
print('Loading...')
 train = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
features = [c for c in train.columns if 'feature' in c]
print('Filling...')
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features] = train[features].fillna(method = 'ffill' ).fillna(0)
 train['action'] =(train['resp'] > 0 ).astype('int')
 print('Finish.' )
def build_model() :
model = Sequential()
model.add(efficientnetb3)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=1e-4,decay=1e-6),
metrics=['accuracy'] 
)
 return model<define_variables>
n_splits = 5
group_gap = 31
bucket_num = 8
div = round(train_df.shape[0]/bucket_num )<create_dataframe>
class GroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_size=None 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_size = max_train_size
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = n_groups // n_folds
group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
for train_group_idx in unique_groups[:group_test_start]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
if self.max_train_size and self.max_train_size < train_end:
train_array = train_array[train_end -
self.max_train_size:train_end]
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 yield [int(i)for i in train_array], [int(i)for i in test_array]
df_init = {
'val_loss': [0.0],
'val_acc': [0.0],
'loss': [0.0],
'acc': [0.0],
'bucket': [0.0]
}
results = pd.DataFrame(df_init )<define_search_space>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
epochs = [5,5,5,5,5,5,5,5]
kappa_metrics = Metrics()
kappa_metrics.val_kappas = []
learn_control = ReduceLROnPlateau(monitor='val_loss',
patience=5,
verbose=1,
factor=.5,
min_lr=1e-7)
 checkpoint = ModelCheckpoint('val_model.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min' )<prepare_output>
def optimise(params):
print(params)
 p = {'learning_rate': params['learning_rate'],
'max_depth': params['max_depth'],
'gamma': params['gamma'],
'min_child_weight': params['min_child_weight'],
'subsample': params['subsample'],
'colsample_bytree': params['colsample_bytree'],
'verbosity': 0,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'tree_method': 'gpu_hist',
'random_state': 42,
}
scores = []
gkf = PurgedGroupTimeSeriesSplit(n_splits = n_splits, group_gap = group_gap)
 for fold,(tr, te)in enumerate(gkf.split(train['action'].values, train['action'].values, train['date'].values)) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
d_tr = xgb.DMatrix(X_tr, y_tr)
 d_val = xgb.DMatrix(X_val, y_val)
 clf = xgb.train(p, d_tr, params['n_round'], [(d_val, 'eval')], verbose_eval = False)
 val_pred = clf.predict(d_val)
 score = roc_auc_score(y_val, val_pred)
 scores.append(score)
 del clf, val_pred, d_tr, d_val, X_tr, X_val, y_tr, y_val, score
rubbish = gc.collect()
score_avg = weighted_average(scores)
 print(score_avg)
 return - score_avg
results = results.iloc[1:]
results['kappa'] = kappa_metrics.val_kappas
results = results.reset_index()
results = results.rename(index=str, columns={"index": "epoch"})
 results<save_to_csv>
%%time
p_best = {'learning_rate': 0.014106988708201764,
'max_depth': 8,
'gamma': 9.800749651802157,
'min_child_weight': 0.3032862674190433,
'subsample': 0.4648851101943981,
'colsample_bytree': 0.994909039539885,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'tree_method': 'gpu_hist',
}
X_tr, y_tr = train.loc[train['date'] > 85, features].values, train.loc[train['date'] > 85, 'action'].values
d_tr = xgb.DMatrix(X_tr, y_tr)
 models = []
for seed in range(5):
p_best['random_state'] = seed
clf = xgb.train(p_best, d_tr, 950)
 models.append(clf)
 rubbish = gc.collect()
results[['loss', 'val_loss']].plot()
results[['acc', 'val_acc']].plot()
results[['kappa']].plot()
results.to_csv('model_results.csv',index=False )<load_pretrained>
@njit
def fast_fillna(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
train.loc[0, features[1:]] = fast_fillna(train.loc[0, features[1:]].values, 0 )
model.load_weights('val_model.h5' )<split>
env = janestreet.make_env()
env_iter = env.iter_test()
x_train, x_val, y_train, y_val = train_test_split(
 x_val, y_val,
test_size=0.2,
random_state=nr_seed 
)
 gc.collect()<define_variables>
opt_th = 0.503
tmp = np.zeros(len(features))
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[0, :] = fast_fillna(x_tt[0, :], tmp)
 tmp = x_tt[0, :]
d_tt = xgb.DMatrix(x_tt)
 pred = 0.
 for clf in models:
pred += clf.predict(d_tt)/ len(models)
 pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )