Kaggle competition: Jane Street Market Prediction
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 tf.random.set_seed(seed)
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_glove_test_y =(pred_glove_test_y>0.35 ).astype(int )<set_options>
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
TRAINING = False
USE_FINETUNE = False
FOLDS = 5
SEED = 1111
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
f_mean = np.mean(train[features[1:]].values,axis=0)
 y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
features1 = [f'feature_{i}' for i in range(71)]
features2 = [f'feature_{i}' for i in range(71, 130)]
assert len(features)== len(features1)+ len(features2)
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
class LiteModel:
@classmethod
def from_file(cls, model_path):
return LiteModel(tf.lite.Interpreter(model_path=model_path))
@classmethod
def from_keras_model(cls, kmodel):
converter = tf.lite.TFLiteConverter.from_keras_model(kmodel)
 tflite_model = converter.convert()
return LiteModel(tf.lite.Interpreter(model_content=tflite_model))
def __init__(self, interpreter):
self.interpreter = interpreter
self.interpreter.allocate_tensors()
input_det = self.interpreter.get_input_details() [0]
output_det = self.interpreter.get_output_details() [0]
self.input_index = input_det["index"]
self.output_index = output_det["index"]
self.input_shape = input_det["shape"]
self.output_shape = output_det["shape"]
self.input_dtype = input_det["dtype"]
self.output_dtype = output_det["dtype"]
def predict(self, inp):
inp = inp.astype(self.input_dtype)
 count = inp.shape[0]
out = np.zeros(( count, self.output_shape[1]), dtype=self.output_dtype)
 for i in range(count):
self.interpreter.set_tensor(self.input_index, inp[i:i+1])
 self.interpreter.invoke()
out[i] = self.interpreter.get_tensor(self.output_index)[0]
return out
def predict_single(self, inp):

 inp = np.array([inp], dtype=self.input_dtype)
 self.interpreter.set_tensor(self.input_index, inp)
 self.interpreter.invoke()
out = self.interpreter.get_tensor(self.output_index)
 return out[0]
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def create_encoder(hp,input_dim,output_dim):
i = Input(input_dim)
 x = BatchNormalization()(i)
 x = GaussianNoise(hp.Float('noise',0.1,0.24,default=0.1))(x)
 encoding_units = []
encoding_dp = []
for j in range(hp.Int('num_encoding_layers',1,3)) :
u = hp.Int(f'encoding_units_{j}',32,64)
 encoding_units.append(u)
 x = Dense(u,activation='relu' )(x)
 x = BatchNormalization()(x)
 dp = hp.Float(f'encoding_dp_{j}',0.1,0.24)
 encoding_dp.append(dp)
 x = Dropout(dp )(x)
 for u,dp in zip(encoding_units[::-1],encoding_dp[::-1]):
x = Dense(u,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(dp )(x)
 decoded = Dense(input_dim,activation='linear',name='decoded' )(x)
 for j in range(hp.Int('num_bottleneck_layers',1,2)) :
x = Dense(hp.Int(f'bottleneck_units_{j}',16,64))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'bottleneck_dp_{j}',0.1,0.242))(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.00071),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)} )
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_fasttext_test_y =(pred_fasttext_test_y>0.33 ).astype(int )<set_options>
def create_resnet(hp, n_features, n_labels, encoder, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,), name = 'Input1')
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(hp.Float('init_dropout',0.1,0.4)) ,
tf.keras.layers.Dense(128, activation = "mish")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_vae = encoder(input_1)[0]
x = tf.keras.layers.Dense(256, activation='relu' )(input_1)
 x = tf.keras.layers.Reshape(( 16, 16))(x)
 x = tf.keras.layers.Conv1D(filters=32,
kernel_size=5,
strides=3,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 input_311 = tf.keras.layers.Flatten()(x)
 input_3_concat = tf.keras.layers.Concatenate()([input_1, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(hp.Float('init_dropout',0.1,0.4)) ,
tf.keras.layers.Dense(128, "mish"),
],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Concatenate()([input_1, input_3, input_4, input_311, input_vae])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(hp.Int('num_units_{i}',64,196), kernel_initializer='lecun_normal', activation='mish'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(hp.Float('init_dropout',0.1,0.4)) ,
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = input_1, outputs = output)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.01,default=0.001)) , loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=['AUC'])
 return model
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
<train_model><EOS> 
if not TRAINING:
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )