Kaggle competition: Jane Street Market Prediction
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 if DEBUG:
train_df = train_df[:100000]
train_df['question_text'] = train_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
DEBUG, train_df.shape, test_df.shape<string_transform>
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam, Adagrad
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import random
import kerastuner as kt
def load_embedding(file, tokenizer):
def split(line):
word, *arr = line.split(' ')
 return word, np.asarray(arr, dtype='float32')
 with open(file, encoding='utf8', errors='ignore')as f:
if DEBUG:
embeddings = dict(split(line)for line in tqdm(itertools.islice(f, 10000)) if len(line)> 100)
 else:
embeddings = dict(split(line)for line in tqdm(f)if len(line)> 100)
 values = np.stack(embeddings.values())
 mean = values.mean()
std = values.std()
n_words = min(MAX_FEATURES, len(tokenizer.word_index))
embedding_matrix = np.random.normal(mean, std,(n_words, EMBED_SIZE))
for word, i in tokenizer.word_index.items() :
if i < MAX_FEATURES and word in embeddings:
embedding_matrix[i] = embeddings[word]
return embedding_matrix<choose_model_class>
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 tf.random.set_seed(seed )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
class BlockingTimeSeriesSplitWithGap() :
def __init__(self,n_splits=5,gap=0,group_fold_size=None):

 self.n_splits = n_splits
self.gap = gap
self.group_fold_size = group_fold_size
def get_n_splits(self,X,y,groups):
return self.n_splits
def split(self,X,groups,y=None,test_size=0.2):

 group_indices = {i:np.where(groups==i)for i in np.unique(groups)}
groups = np.unique(groups)
 if self.group_fold_size is None:
group_fold_size = len(groups)//self.n_splits
elif self.group_fold_size > len(groups):
raise AssertionError(f'group_fold_size={self.group_fold_size} is greater than the number of groups({len(groups)} ).')
 elif self.group_fold_size * self.n_splits < len(groups):
warnings.warn(f'group_fold_size={self.group_fold_size} is too small for a group of {len(groups)} items.Setting group_fold_size={(len(groups)) //self.n_splits}.')
 group_fold_size = len(groups)//self.n_splits
else:
group_fold_size = self.group_fold_size
splits = list(windowed(groups,group_fold_size,step=1))
idx = np.round(np.linspace(0, len(splits)-1, self.n_splits)).astype(int)
 splits = [splits[i] for i in idx]
for idx in splits:
idx = list(filter(None,idx))
mid = int(( 1-test_size)* len(idx))
group_idx_tr, group_idx_te = idx[0: mid], idx[mid + self.gap: -1]
if len(group_idx_te)< 1:
break
tr = np.concatenate([group_indices[gr_i] for gr_i in group_idx_tr],axis=1 ).ravel()
te = np.concatenate([group_indices[gr_i] for gr_i in group_idx_te],axis=1 ).ravel()
yield tr,te
def model_gru_atten_3(embedding):
input_ = Input(shape=(SEQUENCE_LENGTH,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding] )(input_)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(96, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(SEQUENCE_LENGTH )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<train_model>
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )
class AveragedModel:
def __init__(self, tokenizer):
self.embeddings = [
load_embedding(GLOVE, tokenizer),
load_embedding(WIKI, tokenizer),
load_embedding(PARAGRAM, tokenizer),
]
self.models = [model_gru_atten_3(embedding)for embedding in self.embeddings]
def fit(self, *args, **kwargs):
for model in self.models:
model.fit(*args, **kwargs)
 def predict(self, *args, **kwargs):
outputs = [model.predict(*args, **kwargs)for model in self.models]
return np.array(outputs ).mean(axis=0 )<string_transform>
TRAINING = False
USE_FINETUNE = True
FOLDS = 5
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
EPSILON = {c:0.0 for c in resp_cols}
X = train[features].values
y = np.stack([(train[c] > EPSILON[c] ).astype('int')for c in resp_cols] ).T
f64_max = train['feature_64'].values.max()
f64_min = train['feature_64'].values.min()
X = train_df["question_text"]
y = train_df['target'].values
predicted_y = np.zeros_like(y, dtype='float32')
 indices = []
splitted_Xy = []
models = []
splitter = StratifiedKFold(2, random_state=0)
 for train_index, test_index in splitter.split(X, y):
indices.append(( train_index, test_index))
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(X[train_index])
 train_X = tokenizer.texts_to_sequences(X[train_index])
 train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)
 test_X = tokenizer.texts_to_sequences(X[test_index])
 test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)
 train_y = y[train_index]
test_y = y[test_index]
splitted_Xy.append(( train_X, test_X, train_y, test_y))
model = AveragedModel(tokenizer)
 models.append(model )<train_model>
def calculate_vif(X,thresh=5.0,sample_size=-1,features=None):
if sample_size<0:
new_X = X.copy()
else:
new_X = X[np.random.choice(len(X), size=sample_size)]
if features and len(features)== new_X.shape[1]:
features_copy = features.copy()
else:
features_copy = list(range(new_X.shape[-1]))
dropped_idx = []
dropped = True
while dropped:
dropped = False
cc = np.corrcoef(new_X, rowvar=False)
 VIF = np.linalg.inv(cc ).diagonal()
maxloc = np.argmax(VIF)
 if max(VIF)> thresh:
print(f'Dropped {features_copy[maxloc]} with vif={max(VIF)}')
 features_copy.pop(maxloc)
 new_X = np.delete(new_X, maxloc, axis=1)
 dropped_idx.append(maxloc)
 dropped = True
del new_X
for idx in dropped_idx:
X = np.delete(X,idx,axis=1)
 return X, features_copy
for model,(train_X, test_X, train_y, test_y),(train_index, test_index)in zip(models, splitted_Xy, indices):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(test_X, test_y), verbose=0)
 yp = model.predict([test_X], batch_size=512, verbose=0 ).flatten()
predicted_y[test_index] = yp<compute_test_metric>
gc.collect()
threshold = 0.5
confusion_matrix(y,(predicted_y > threshold ).astype(int))<compute_test_metric>
if TRAINING:
pd.to_pickle(features,'features.pkl')
 else:
features = pd.read_pickle('.. /input/jsautoencoder/features.pkl')
 f_mean = f_mean = np.mean(train[features[1:]].values,axis=0 )
f1_score(y, predicted_y > threshold )<find_best_params>
optimizer = lambda x: Adam(x )
def find_best_threshold(y, predicted_y):
best_f1, best_threshold = 0, 0
for threshold in np.arange(0.01, 0.99, 0.01):
f1 = f1_score(y, predicted_y > threshold)
 if f1 > best_f1:
best_f1, best_threshold = f1, threshold
return best_f1, best_threshold
best_f1, best_threshold = find_best_threshold(y, predicted_y)
 best_f1, best_threshold<find_best_model_class>
def create_encoder(hp,input_dim,output_dim):
i = Input(input_dim)
 x = BatchNormalization()(i)
 x = GaussianNoise(hp.Float('noise',0.1,0.5,default=0.1))(x)
 encoding_units = []
encoding_dp = []
for j in range(hp.Int('num_encoding_layers',1,3)) :
u = hp.Int(f'encoding_units_{j}',64,512)
 encoding_units.append(u)
 x = BatchNormalization()(x)
 x = Dense(u,activation='relu' )(x)
 dp = hp.Float(f'encoding_dp_{j}',0.1,0.5)
 encoding_dp.append(dp)
 x = Dropout(dp )(x)
 for u,dp in zip(encoding_units[::-1],encoding_dp[::-1]):
x = BatchNormalization()(x)
 x = Dense(u,activation='relu' )(x)
 x = Dropout(dp )(x)
 decoded = Dense(input_dim,activation='linear',name='decoded' )(x)
 for j in range(hp.Int('num_bottleneck_layers',1,3)) :
x = BatchNormalization()(x)
 x = Dense(hp.Int(f'bottleneck_units_{j}',16,64))(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'bottleneck_dp_{j}',0.1,0.5))(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=optimizer(1e-3),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder
rocs = [(fpr, tpr, thresholds, roc_auc, best_f1, best_threshold)]
for i in range(3):
for model,(train_X, test_X, train_y, test_y),(train_index, test_index)in zip(models, splitted_Xy, indices):
model.fit(train_X, train_y, batch_size=1024, epochs=1, validation_data=(test_X, test_y), verbose=0)
 yp = model.predict([test_X], batch_size=1024, verbose=0 ).flatten()
predicted_y[test_index] = yp
fpr_, tpr_, thresholds_ = roc_curve(y, predicted_y, pos_label=1)
 roc_auc_ = auc(fpr_, tpr_)
 best_f1, best_threshold = find_best_threshold(y, predicted_y)
 rocs.append(( fpr_, tpr_, thresholds_, roc_auc_, best_f1, best_threshold))
print(f'Epoch {i + 2}: ROC_AUC={roc_auc_}')
 print('Confusion matrix', confusion_matrix(y,(predicted_y > best_threshold ).astype(int)))
 rocs<find_best_params>
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)[0]
x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.1,0.5))(x)
 for i in range(hp.Int('num_layers',3,6)) :
x = BatchNormalization()(x)
 x = Dense(hp.Int(f'num_units_{i}',16,64))(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.1,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=optimizer(1e-3),loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
aucs = [i[3] for i in rocs]
argmax = np.argmax(aucs)
 best_epochs = argmax + 1
best_threshold = rocs[argmax][5]
aucs, best_epochs, aucs[argmax], best_threshold<import_modules>
def utility_score_bincount(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
bias = 0.1
def time_bias(f0,f64):
if f0 < 0:
correction =(1+bias/2)- bias *(f64-f64_min)/(f64_max-f64_min)
 else:
correction =(1-bias/2)+ bias *(f64-f64_min)/(f64_max-f64_min)
 return correction
<split><EOS> 
if not TRAINING:
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
f0 = test_df['feature_0'].item()
f64 = test_df['feature_64'].item()
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt,training=False ).numpy() for model in models],axis=0 ).squeeze()
tb = time_bias(f0,f64)
 pred = np.mean(pred)* tb
pred_df.action = np.where(pred > th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )