Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<create_dataframe>
import os
import pandas as pd
import missingno as msn
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
import datatable as dt
import seaborn as sns
from numba import njit
train_dataset = tf.data.Dataset.from_tensor_slices(( X_train, y_train))<create_dataframe>
ccut = 0.975
dropout = 0.2
dateCut = 'date > 85'
nepochs = 140
print("Running prediction with correlation cutoff: ", ccut)
 print('Layer dropouts set to: ', dropout)
 runjane = True
validation_dataset = tf.data.Dataset.from_tensor_slices(( X_test, y_test))<define_variables>
if runjane :
print('
total_size = 0
start_path = '.. /input/jane-street-market-prediction'
for path, dirs, files in os.walk(start_path):
for f in files:
fp = os.path.join(path, f)
 total_size += os.path.getsize(fp)
 print("Directory size: " + str(round(total_size/ 1000000, 2)) + 'MB')
 train_raw = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 else :
train_raw = pd.read_csv('train85_100k.csv')
 print("Data is loaded" )
train_examples_batch, train_labels_batch = next(iter(train_dataset.batch(10)))
 train_examples_batch<categorify>
print("Reading in data with cutoff: ", dateCut)
 train_raw=train_raw.query(dateCut ).reset_index(drop = True )
embedding = "https://tfhub.dev/google/universal-sentence-encoder-large/5"
hub_layer = hub.KerasLayer(embedding, input_shape=[],
dtype=tf.string, trainable=False)
 hub_layer(tf.convert_to_tensor(train_examples_batch[:3]))<choose_model_class>
correl= train_raw[features].corr().abs()
correl[correl == 1] = 0
for i in range(len(correl)) :
for j in range(i):
correl.iat[j,i] = 0
cflist = correl.unstack().sort_values(ascending=False ).drop_duplicates()
cflcut = cflist[cflist > ccut]
cflcutis = cflcut.index
print("List of correlated features with absolute value > ", ccut)
 feat2drop = []
print("Building list of features to drop")
 for i in range(len(cflcutis)) :
print("Index: %s , value: %f " %(cflcutis[i] , cflcut[i]))
if not(cflcutis[i][1] in feat2drop):
feat2drop.append(cflcutis[i][1])
 print("Features to drop: ", len(feat2drop))
print(feat2drop)
 for feat in feat2drop:
features.remove(feat)
 print("Number of remaining features is: ", len(features))
model = tf.keras.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.summary()<choose_model_class>
print("Building model and stetting up training now!")
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'] )<train_model>
train = train_raw.fillna(train_raw.mean())
 print(train.head())
 print("Null values found in train: ", train.isnull().sum().sum())
 train = reduce_mem_usage(train )
history = model.fit(train_dataset.shuffle(10000 ).batch(512),
epochs=40,
validation_data=validation_dataset.batch(512),
verbose=1 )<predict_on_test>
train = train[train['weight'] != 0]
if 'resp' in train.columns :
train['action'] = np.where(train['resp'] > 0,1,0)
 else:
train['action'] = np.where(train['weight'] > 0,1,0)
 X = train.loc[:, features]
y = train.loc[:, 'action']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
y_pred_list = []
for text in X_test_lst:
y_pred = model.predict(tf.expand_dims(text, 0))
y_pred_list.append(y_pred.tolist() [0] )<concatenate>
X_nptrain = X_train[:200000].to_numpy()
y_nptrain = y_train[:200000].to_numpy()
print("Shape X_train: ", X_train.shape[1])
 dim_lay1 = X_train.shape[1]
dim_lay2 = dim_lay1 / 1.5
dim_lay3 = dim_lay2 / 2
dim_lay4 = min(10,dim_lay3/2 )
y_pred_boolean = tf.greater(y_pred_list,.7 )<define_variables>
normalizer = preprocessing.Normalization()
normalizer.adapt(X_nptrain)
 input_shape = X_train.shape[1:]
final=[]
for data in y_pred_boolean:
if data == True:
final.append(1)
 else:
final.append(0 )<create_dataframe>
inputs = keras.Input(shape=input_shape)
 x = tf.keras.layers.BatchNormalization()(inputs)
 x = layers.Dense(dim_lay1, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay2, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay3, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay4, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 outputs = layers.Dense(1, activation="sigmoid" )(x)
 model = keras.Model(inputs, outputs)
 model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
output = pd.DataFrame({'id':test['id'],'target':final} )<save_to_csv>
%time history = model.fit(X_train, y_train, epochs=nepochs, batch_size = 4096, validation_split=0.05, verbose=1 )
output.to_csv("output.csv",index=False )<load_from_url>
score = model.evaluate(X_test, y_test)
 print("Accuracy Score on X_test: "+str(round(score[1],4)) )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
if runjane :
janestreet.make_env.__called__ = False
env = janestreet.make_env()
iter_test = env.iter_test()
<categorify><EOS> 
%%time
if runjane :
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() > 0:
X_test = test_df.loc[: , features].values
X_test = np.nan_to_num(X_test[:,:])
 y_preds = model(X_test)
 sample_prediction_df.action = np.where(y_preds >= 0.5, 1, 0 ).astype(int)
 else:
sample_prediction_df.action = 0
submission=env.predict(sample_prediction_df)
 print("test_df prediction completed!")
 print(sample_prediction_df)