Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<create_dataframe>
import janestreet
from tqdm.notebook import tqdm
import time
import numpy as np
import os ,gc
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import datatable as dt
import tensorflow as tf
from tensorflow import keras
import tensorflow_addons as tfa
missings_count = {col:df_train_pop[col].isnull().sum() for col in df_train_pop.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))<create_dataframe>
def set_seed(seed=7):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 set_seed(seed=7)
 plt.style.use('seaborn' )
df_fin_selc = df_train_pop[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','Day_num','Day','Month','Year','Population']]<feature_engineering>
%time
train_df=dt.fread('.. /input/jane-street-market-prediction/train.csv' ).to_pandas().query('weight > 0 and date >85' ).reset_index(drop=True)
 test_df=dt.fread('.. /input/jane-street-market-prediction/example_test.csv' ).to_pandas().reset_index(drop=True)
 sample_submission=pd.read_csv('.. /input/jane-street-market-prediction/example_sample_submission.csv')
df_fin_selc['I'] = df_fin_selc['ConfirmedCases']-df_fin_selc['Fatalities']
df_fin_selc['R'] = df_fin_selc['Fatalities']
df_fin_selc['S'] = df_fin_selc['Population']-df_fin_selc['ConfirmedCases']
df_SIR = df_fin_selc[['Province_State','Country_Region','Population','Day_num','S','I','R']].reindex()
df_SIR[df_SIR['Country_Region'] == 'Italy'].tail(5 )<import_modules>
print('Training set shape {}
Test set shape {} '.format(train_df.shape,test_df.shape))
import numpy as np
from scipy.integrate import odeint
from scipy import integrate, optimize<compute_test_metric>
null_val_ratio= train_df.isnull().sum().sum() /np.product(train_df.shape)
 print(f'% of null values in train set is {null_val_ratio *100} ')
 train_df=train_df.interpolate(limit_direction='forward',axis=0)
 train_df=train_df.bfill()
gc.collect()
def SIR_testmodel(y,t,bta,gmma):
S, I, R = y
dS_dt = -1*bta*I*S/N
dI_dt =(bta*I*S/N)- gmma*I
dR_dt = gmma*I
return([dS_dt, dI_dt, dR_dt] )<define_search_space>
features = train_df.columns[train_df.columns.str.contains('feature')]
train_df['action']=(train_df['resp']>0 ).astype('int')
 X_train=train_df[features]
y_train=np.stack([(train_df[c] > 0 ).astype('int')for c in resp_cols] ).T
gc.collect()
N = 1
S00 = 0.9
I00 = 0.1
R00 = 0.0
bta = 0.35
gmma = 0.1
t = np.linspace(0,100,1000)
 sol = odeint(SIR_testmodel,[S00,I00,R00],t,args =(bta,gmma))
sol = np.array(sol )<filter>
def Build_model(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'] )<filter>
epochs = 200
batch_size = 8192
hidden_units = [256,128,128,64]
dropout_rates = [0.2,0.25, 0.25, 0.2, 0.15]
label_smoothing = 1e-2
learning_rate = 1e-2
tf.keras.backend.clear_session()
model2 = Build_model(len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate)
 reduce_lr = keras.callbacks.ReduceLROnPlateau(
 monitor = 'AUC', factor = 0.75, patience = 3, verbose = 0, min_delta = 1e-3, mode = 'max',min_lr=1e-8)
 early_stopping=keras.callbacks.EarlyStopping(patience=15,min_delta=1e-2,monitor='loss',restore_best_weights=True)
 model2.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,callbacks=[reduce_lr,early_stopping])
 gc.collect()
Country_Dates[Country_Dates['Country_Region'] == 'Italy']<split>
epochs = 200
batch_size = 8192
hidden_units = [256,128,64]
dropout_rates = [0.2,0.25, 0.20, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-1
tf.keras.backend.clear_session()
model3= Build_model(len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate)
 reduce_lr = keras.callbacks.ReduceLROnPlateau(
 monitor = 'AUC', factor = 0.50, patience = 3, verbose = 0, min_delta = 1e-3, mode = 'max',min_lr=1e-8)
 early_stopping=keras.callbacks.EarlyStopping(patience=15,min_delta=1e-3,monitor='loss',restore_best_weights=True)
 model3.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,callbacks=[reduce_lr,early_stopping])
 gc.collect()
cond1 = Italy_SIR_df.I >= 1
test = Italy_SIR_df['Day_num'][(cond1)]
test
Italy_SIR_df.iloc[:10]<prepare_x_and_y>
models=[model2,model3]
th = 0.491
f = np.median
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
test_df=test_df.interpolate(limit_direction='forward',axis=0)
 test_df=test_df.bfill()
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred=f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )