Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<define_variables>
!pip --quiet install.. /input/treelite/treelite-0.93-py3-none-manylinux2010_x86_64.whl
vote_list = [voting([int(avg_res_xlf[idx] + 0.5),
int(avg_res_svr[idx] + 0.5),
int(avg_res_cb[idx] + 0.5)])
 for idx in range(len(avg_res_xlf)) ]<save_to_csv>
!pip --quiet install.. /input/treelite/treelite_runtime-0.93-py3-none-manylinux2010_x86_64.whl
test_df.diagnosis = vote_list
test_df.hist()
plt.show()
test_df.to_csv('submission.csv',index=False )<count_values>
import numpy as np
import pandas as pd
import os
import treelite
import treelite_runtime
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
import pathlib
from tqdm import tqdm
from random import choices
import operator
import xgboost as xgb
test_df['diagnosis'].value_counts()<train_model>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )
dur = time.time() -start
print('Whole procedure takes {} seconds long...'.format(dur))
print(dur/3600)
 print('done' )<set_options>
SEED = 2021
START_DATE = 86
FOLDS = 5
NN_NAME = 'mlp'
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 set_random_seed(0)
 seed = 0
seed_everything(seed)
 %matplotlib inline
sns.set(style="whitegrid")
 warnings.filterwarnings("ignore")
 sys.path.append(os.path.abspath('.. /input/efficientnet/efficientnet-master/efficientnet-master/')) 
<feature_engineering>
%%time
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
hold_out_set = pd.read_csv('.. /input/aptos-data-split/hold-out.csv')
 X_train = hold_out_set[hold_out_set['set'] == 'train']
X_val = hold_out_set[hold_out_set['set'] == 'validation']
test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print('Number of train samples: ', X_train.shape[0])
 print('Number of validation samples: ', X_val.shape[0])
 print('Number of test samples: ', test.shape[0])
 X_train["id_code"] = X_train["id_code"].apply(lambda x: x + ".png")
 X_val["id_code"] = X_val["id_code"].apply(lambda x: x + ".png")
 test["id_code"] = test["id_code"].apply(lambda x: x + ".png")
 display(X_train.head() )<define_variables>
predictor = treelite_runtime.Predictor('.. /input/janestreet-faster-inference-by-xgb-with-treelite/mymodel.so', verbose=True )
HEIGHT = 224
WIDTH = 224
CHANNELS = 3
TTA_STEPS = 10
weights_path_list = ['.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold1.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold2.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold3.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold4.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold5.h5']<compute_test_metric>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']
def plot_confusion_matrix(train, validation, labels=labels):
train_labels, train_preds = train
validation_labels, validation_preds = validation
fig,(ax1, ax2)= plt.subplots(1, 2, sharex='col', figsize=(24, 7))
train_cnf_matrix = confusion_matrix(train_labels, train_preds)
 validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)
 train_cnf_matrix_norm = train_cnf_matrix.astype('float')/ train_cnf_matrix.sum(axis=1)[:, np.newaxis]
validation_cnf_matrix_norm = validation_cnf_matrix.astype('float')/ validation_cnf_matrix.sum(axis=1)[:, np.newaxis]
train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)
 validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)
 sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap="Blues",ax=ax1 ).set_title('Train')
 sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2 ).set_title('Validation')
 plt.show()
def evaluate_model(train, validation):
train_labels, train_preds = train
validation_labels, validation_preds = validation
print("Train Cohen Kappa score: %.3f" % cohen_kappa_score(train_preds, train_labels, weights='quadratic'))
print("Validation Cohen Kappa score: %.3f" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))
print("Complete set Cohen Kappa score: %.3f" % cohen_kappa_score(np.append(train_preds, validation_preds), np.append(train_labels, validation_labels), weights='quadratic'))
def classify(x):
if x < 0.5:
return 0
elif x < 1.5:
return 1
elif x < 2.5:
return 2
elif x < 3.5:
return 3
return 4
def ensemble_preds(model_list, generator):
preds_ensemble = []
for model in model_list:
generator.reset()
preds = model.predict_generator(generator, steps=generator.n)
 preds_ensemble.append(preds)
 return np.mean(preds_ensemble, axis=0)
 def apply_tta(model, generator, steps=5):
step_size = generator.n//generator.batch_size
preds_tta = []
for i in range(steps):
generator.reset()
preds = model.predict_generator(generator, steps=step_size)
 preds_tta.append(preds)
 return np.mean(preds_tta, axis=0)
 def test_ensemble_preds(model_list, generator, steps=5):
preds_ensemble = []
for model in model_list:
preds = apply_tta(model, generator, steps)
 preds_ensemble.append(preds)
 return np.mean(preds_ensemble, axis=0 )<create_dataframe>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-1dcnn-for-feature-extraction-train/encoder.hdf5')
 encoder.trainable = False
datagen=ImageDataGenerator(rescale=1./255,
rotation_range=360,
horizontal_flip=True,
vertical_flip=True)
 train_generator=datagen.flow_from_dataframe(
 dataframe=X_train,
directory=train_dest_path,
x_col="id_code",
y_col="diagnosis",
class_mode="raw",
batch_size=1,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed)
 valid_generator=datagen.flow_from_dataframe(
 dataframe=X_val,
directory=validation_dest_path,
x_col="id_code",
y_col="diagnosis",
class_mode="raw",
batch_size=1,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed)
 test_generator=datagen.flow_from_dataframe(
 dataframe=test,
directory=test_dest_path,
x_col="id_code",
batch_size=1,
class_mode=None,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed )<choose_model_class>
def create_1dcnn(input_dim, output_dim, encoder):
inputs = tf.keras.layers.Input(input_dim)
 x = encoder(inputs)
 x = tf.keras.layers.Concatenate()([x,inputs])
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(4096, activation='relu' )(x)
 x = tf.keras.layers.Reshape(( 256, 16))(x)
 x = tf.keras.layers.Conv1D(filters=16,
kernel_size=7,
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 for i in range(2):
x = tf.keras.layers.Dense(256 //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(0.01 )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-02)
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
def create_model(input_shape, weights_path):
input_tensor = Input(shape=input_shape)
 base_model = EfficientNetB5(weights=None,
include_top=False,
input_tensor=input_tensor)
 x = GlobalAveragePooling2D()(base_model.output)
 final_output = Dense(1, activation='linear', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 model.load_weights(weights_path)
 return model<train_model>
def create_resnet(n_features, n_labels, encoder, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,))
input_2 = encoder(input_1)
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(512, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1), name='l2_norm'),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, ], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model
model_list = []
for weights_path in weights_path_list:
model_list.append(create_model(input_shape=(HEIGHT, WIDTH, CHANNELS), weights_path=weights_path))<predict_on_test>
%%time
if NN_NAME == '1dcnn':
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_1dcnn(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-1dcnn-for-feature-extraction-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
 models = [models[-1]]
preds_ensemble = ensemble_preds(model_list, train_generator)
 preds_ensemble = [classify(x)for x in preds_ensemble]
train_preds = pd.DataFrame({'label':train_generator.labels, 'pred':preds_ensemble})
 preds_ensemble = ensemble_preds(model_list, valid_generator)
 preds_ensemble = [classify(x)for x in preds_ensemble]
validation_preds = pd.DataFrame({'label':valid_generator.labels, 'pred':preds_ensemble} )<compute_test_metric>
%%time
if NN_NAME == 'resnet':
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_resnet(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-resnet-with-autoencoder-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
 models = [models[-1]]
evaluate_model(( train_preds['label'], train_preds['pred']),(validation_preds['label'], validation_preds['pred']))<prepare_output>
%%time
if NN_NAME == 'mlp':
model = tf.keras.models.load_model('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 models = [model]
<load_pretrained><EOS> 
f = np.median
th = 0.500
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
batch = treelite_runtime.Batch.from_npy2d(x_tt)
 xgb_pred = predictor.predict(batch)
 if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(0.9*pred + 0.1*xgb_pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )