Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<train_on_grid>
import os, logging,math
from collections import deque
from tqdm import tqdm
import numpy as np
import pandas as pd
from sklearn import metrics
import datatable as dt
from datatable import f
import mxnet as mx
from mxnet import nd, autograd, gluon
from mxnet.gluon import nn
def preprocess_image(image_path, width=320, height=320, new_data=False):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = circle_crop(img)
 img = cv2.resize(img,(width,height))
return img<prepare_x_and_y>
class GroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_size=None 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_size = max_train_size
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = n_groups // n_folds
group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
for train_group_idx in unique_groups[:group_test_start]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
if self.max_train_size and self.max_train_size < train_end:
train_array = train_array[train_end -
self.max_train_size:train_end]
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 yield [int(i)for i in train_array], [int(i)for i in test_array]
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
N = val_df.shape[0]
x_val = np.empty(( N, HEIGHT, WIDTH, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(val_df['id_code'])) :
x_val[i, :, :, :] = preprocess_image(
 f'{image_id}',
height=HEIGHT, width=WIDTH, new_data=True
)<categorify>
class MLPModel(nn.Block):
def __init__(self, hiddenList, dropoutList,actType='relu', **kwargs):
super(MLPModel, self ).__init__(**kwargs)
 assert len(hiddenList)== len(dropoutList)
 assert actType in ['relu','tanh','softrelu']
self.net=nn.Sequential()
with self.name_scope() :
self.embedding1 = nn.Embedding(2,2)
 for nHidden, dropout in zip(hiddenList, dropoutList):
self.net.add(nn.Dense(nHidden, activation=actType))
self.net.add(nn.Dropout(dropout))
self.Resp = nn.Dense(1, activation='sigmoid')
 self.Resp1 = nn.Dense(1, activation='sigmoid')
 self.Resp2 = nn.Dense(1, activation='sigmoid')
 self.Resp3 = nn.Dense(1, activation='sigmoid')
 self.Resp4 = nn.Dense(1, activation='sigmoid')
 def forward(self, X):
embed1 = self.embedding1(X[:,0])
 embedConcat = nd.concat(embed1,X[:,1:])
 tmpOutput = self.net(embedConcat)
 resp = self.Resp(tmpOutput)
 resp1 = self.Resp1(tmpOutput)
 resp2 = self.Resp2(tmpOutput)
 resp3 = self.Resp3(tmpOutput)
 resp4 = self.Resp4(tmpOutput)
 return resp[:,0],resp1[:,0],resp2[:,0],resp3[:,0],resp4[:,0]
y_train = pd.get_dummies(train_df['diagnosis'] ).values
y_val = pd.get_dummies(val_df['diagnosis'] ).values
print(y_train.shape)
 print(x_val.shape)
 print(y_val.shape )<categorify>

 dataDirPath = "/kaggle/input/jane-street-market-prediction/"
train = dt.fread(dataDirPath + 'train.csv', na_strings = ['NA','NULL','NaN','\\N'])
 train = train[(f.date>85)&(f.weight>0),:]
train[:,["action1","action2","action3","action4","action"]] = train[:, [f.resp_1>0, f.resp_2>0, f.resp_3>0, f.resp_4>0, f.resp>0]]
train[["action1","action2","action3","action4","action"]] = dt.int32
features = [c for c in train.names if 'feature' in c]
resps = [c for c in train.names if 'resp' in c]
actions = [c for c in train.names if 'action' in c]
meanImputeValues = train[:, 8:137].mean().to_numpy() [0,:]
train = train.to_pandas()
train = train.astype({c: np.float32 for c, t in train.dtypes.items() if t == np.float64})
 train = train.fillna(train.mean())
 train.loc[train['feature_0'] ==-1, 'feature_0'] = 0
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 y_val_multi = np.empty(y_val.shape, dtype=y_val.dtype)
 y_val_multi[:, 4] = y_val[:, 4]
for i in range(3, -1, -1):
y_val_multi[:, i] = np.logical_or(y_val[:, i], y_val_multi[:, i+1])
 print("Y_train multi: {}".format(y_train_multi.shape))
print("Y_val multi: {}".format(y_val_multi.shape))<prepare_x_and_y>
nSplits = 5
groupGap = 5
cv = PurgedGroupTimeSeriesSplit(
 n_splits=5,
max_train_group_size=np.inf,
group_gap=0,
max_test_group_size=np.inf 
)
 foldList = []
trainXList = []; trainYList = []; trainWList = []
validXList = []; validYList = []; validWList =[];validDList = [];
for fold,(tr, te)in enumerate(cv.split(train['resp'].values, train['resp'].values, train['date'].values)) :
if fold != 4: continue
subTrain, subValid = train.loc[tr,], train.loc[te,]
trainX, trainY, trainW = subTrain.loc[tr, features].values, subTrain.loc[tr, resps].values, subTrain.loc[tr, 'weight'].values
validX, validY, validW,validD = subValid.loc[te, features].values, subValid.loc[te, resps].values, subValid.loc[te, 'weight'].values, subValid.loc[te, 'date'].values
foldList.append(fold)
 trainXList.append(trainX); trainYList.append(trainY); trainWList.append(trainW)
 validXList.append(validX); validYList.append(validY); validWList.append(validW); validDList.append(validD )
y_train = y_train_multi
y_val = y_val_multi<drop_column>
hiddenList = [128, 64,64]
dropoutList = [0.2, 0.3,0.3]
baseMLP = MLPModel(hiddenList,dropoutList)
 modelPrefix = 'MLP_{}d{}l.params'.format(hiddenList[0], len(hiddenList))
modelCtx = mx.gpu() if mx.context.num_gpus() else mx.cpu()
dataCtx = mx.gpu() if mx.context.num_gpus() else mx.cpu()
mlpTrainer = MLPTrainer(baseMLP, dataCtx, modelCtx)

epochs = 50
esEpochs = 3
evalCriteria = 'max'
batchSize = 128*8
learningRate = 0.001
sampleRate = 0.8
smoothingAlpha=0.005
initializer = mx.init.Xavier(magnitude=2.24)
 optimizer = 'adam';
lossFunc = SigmoidBinaryCrossEntropyLoss(from_sigmoid=True)
 trainerParamsList = {'epochs': epochs, 'esEpochs': esEpochs, 'evalCriteria': evalCriteria,
'batchSize': batchSize, 'learningRate': learningRate, 'sampleRate': sampleRate, 'smoothingAlpha':smoothingAlpha,
'initializer': initializer, 'optimizer':optimizer, 'lossFunc': lossFunc}
del new_train
del old_train
del val_df
gc.collect()<compute_train_metric>

 modelList = []
for fold in foldList:
fold=0
subMark = "fold"+str(fold)
 trainX, trainY, trainW = np.array(trainXList[fold]), np.array(trainYList[fold]),np.array(trainWList[fold])
 testX, testY, testW,testD = np.array(validXList[fold]), np.array(validYList[fold]), np.array(validWList[fold]), np.array(validDList[fold])
 subFit = mlpTrainer.fit(subMark, trainerParamsList, trainX, trainY, trainW, testX, testY, testW,testD)
 modelList.append(subFit)
 break
<define_variables><EOS> 
env = janestreet.make_env()
envIter = env.iter_test()
optThreshold = 0.5
for(testDf, predDf)in tqdm(envIter):
if testDf['weight'].item() > 0:
testDf.loc[testDf['feature_0'] == -1, 'feature_0'] = 0
xTT = testDf.loc[:, features].values
if np.isnan(xTT[:, 1:].sum()):
xTT[:, 1:] = np.nan_to_num(xTT[:, 1:])+ np.isnan(xTT[:, 1:])* meanImputeValues
pred = 0.
 pred = subFit(nd.array(xTT,ctx=dataCtx)) [0].asnumpy()
predDf.action = np.where(pred >= optThreshold, 1, 0 ).astype(int)
 else:
predDf.action = 0
env.predict(predDf)