Kaggle competition: Jane Street Market Prediction
HEIGHT = 256
WIDTH = 256
COEFF = [0.52022015, 1.46022145, 2.49058373, 3.30146459]<choose_model_class>
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import random
import kerastuner as kt
def build_model() :
efficientnetb3 = EfficientNetB4(
 weights=None,
input_shape=(HEIGHT,WIDTH,3),
include_top=False 
)
 model = Sequential()
model.add(efficientnetb3)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(5, activation=elu))
model.add(layers.Dense(1, activation="linear"))
return model<load_pretrained>
class PurgedGroupTimeSeriesSplit(_BaseKFold):
@_deprecate_positional_args
def __init__(self, n_splits = 5, *, max_train_group_size=np.inf, max_test_group_size=np.inf, group_gap=None, verbose=False):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def data_split(self, X, y=None, groups=None):
if groups is None:
raise ValueError("The 'groups' should not be void")
 X, y, groups = indexable(X,Y,groups)
 n_sample = n_sample(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_gap = self.max_test_group_gap
max_train_group_gap = self.max_train_group_gap
n_folds = n_splits + 1
group_dictionary = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
model1 = build_model()
model1.load_weights('.. /input/fork-of-efficientnetb3-regression-keras-2/val_model.h5')
 model1.summary()<load_from_csv>
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=64, epochs=5,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train, validation_data=(X_test,y_test), epochs=epochs, batch_size=batch_size, callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )
TEST_IMG_PATH = '.. /input/aptos2019-blindness-detection/test_images/'
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(test_df.shape)
 original_names = test_df['id_code'].values
test_df['id_code'] = test_df['id_code'] + ".png"
test_df['diagnosis'] = np.zeros(test_df.shape[0])
 display(test_df.head() )<define_variables>
TRAINING = True
USE_FINETUNE = True
FOLDS = 5
SEED = 123
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
tta_steps = 10
predictions1 = []
for i in tqdm(range(tta_steps)) :
test_generator = ImageDataGenerator(rescale=1./255,
horizontal_flip=True,
rotation_range= 90,
vertical_flip=True,
brightness_range=(0.5,1.5),
zoom_range= 0.2,
fill_mode='constant',
preprocessing_function=preprocess_image,
cval = 0 ).flow_from_dataframe(test_df,
x_col='id_code',
y_col = 'diagnosis',
directory = TEST_IMG_PATH,
target_size=(WIDTH, HEIGHT),
batch_size=1,
class_mode='other',
shuffle = False)
 preds1 = model1.predict_generator(test_generator, steps = test_df.shape[0])
 predictions1.append(preds1)
 del test_generator
gc.collect()
pred_test1 = np.mean(predictions1, axis=0)
<prepare_x_and_y>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.summary()
autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
y_test = pred_test1<feature_engineering>
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,3)) :
x = Dense(hp.Int('num_units_{i}',64,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
for i, pred in enumerate(y_test):
if pred < COEFF[0]:
y_test[i] = 0
elif pred >= COEFF[0] and pred < COEFF[1]:
y_test[i] = 1
elif pred >= COEFF[1] and pred < COEFF[2]:
y_test[i] = 2
elif pred >= COEFF[2] and pred < COEFF[3]:
y_test[i] = 3
else:
y_test[i] = 4<feature_engineering>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights(f'.. /input/janestreetencoder/encoder.hdf5')
 encoder.trainable = False
test_df['diagnosis'] = y_test.astype(int)
 test_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')
 test_df.head()<load_from_csv>
def utility_score_bincount(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u
leaks = pd.read_csv('.. /input/aptos-trained-weights/leaks.csv', dtype= {'diagnosis': np.int32})
 codes = leaks['id_code'].values
diagnosis = leaks['diagnosis'].values
for i in range(codes.shape[0]):
test_df.loc[test_df['id_code'] == codes[i], 'diagnosis'] = int(diagnosis[i] )<save_to_csv>
if not TRAINING:
f = np.median
models = models[-2:]
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.502
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )
<define_variables><EOS>