Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<load_from_csv>
Image(".. /input/tf-model-garden-official-models/TF.png" )
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_subm = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<create_dataframe>
import tensorflow as tf
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow_addons as tfa
from sklearn.model_selection import StratifiedKFold
import numpy as np
import pandas as pd
from tqdm import tqdm
import seaborn as sns
import matplotlib.pyplot as plt
df_train_copy = df_train.copy()
df_test_copy = df_test.copy()<data_type_conversions>
NFOLDS = 5
train_all = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train_all = train_all[train_all.date > 85].reset_index(drop = True)
 train_all = train_all[train_all['weight'] != 0]
train_all.fillna(train_all.mean() ,inplace=True )
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )<prepare_x_and_y>
train_all['date_bin'] =(pd.qcut(train_all['date'], q=4, labels=False)+1)*train_all['feature_0']
features = [c for c in train_all.columns if "feature" in c]
f_mean = np.mean(train_all[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train_all.loc[:, train_all.columns.str.contains('feature|date_bin')]
y_train = pd.DataFrame(np.stack([(train_all[c] > 0 ).astype('int')for c in resp_cols] ).T, columns = resp_cols )
y1_df_Train = df_train.iloc[:, -2]
y2_df_Train = df_train.iloc[:, -1]
NOVAL = "NOVAL"
def handlenullstate(State, Country):
if State == NOVAL: return Country
return State<data_type_conversions>
skf = StratifiedKFold(n_splits=NFOLDS, shuffle = True, random_state = 42)
 result = next(skf.split(X_train, X_train.date_bin), None)
 train = train_all.iloc[result[0]].reset_index(drop=True)
 valid = train_all.iloc[result[1]].reset_index(drop=True )
df_train['State'].fillna(NOVAL, inplace=True)
 df_train['State'] = df_train.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_train.loc[:, 'Date'] = df_train.Date.dt.strftime("%m%d")
 df_train["Date"] = df_train["Date"].astype(int)
 df_test['State'].fillna(NOVAL, inplace=True)
 df_test['State'] = df_test.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_test.loc[:, 'Date'] = df_test.Date.dt.strftime("%m%d")
 df_test["Date"] = df_test["Date"].astype(int)
 df_test.head()<define_variables>
del train, valid, train_all, result
covid_pop_taken = covid_pop[['Province.State','Country.Region','Population']]<rename_columns>
MNAME = 'model'
def get_callbacks(idx):
mc = ModelCheckpoint(MNAME+"-{}.h5".format(idx), save_best_only=True)
 rp = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.00001)
 es = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=False)
 return [mc, rp, es]
def create_dnn(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 return model
covid_pop_taken.rename(columns={'Province.State':'State'}, inplace=True)
 covid_pop_taken.rename(columns={'Country.Region':'Country'}, inplace=True )<feature_engineering>
skf = StratifiedKFold(n_splits=NFOLDS, shuffle = True, random_state = 42)
 history = []
for i in range(NFOLDS):
print('fold {}'.format(i))
result = next(skf.split(X_train, X_train.date_bin), None)
 X_tr = X_train.iloc[result[0]].reset_index(drop=True)
 X_tr.drop(labels='date_bin', axis = 1, inplace=True)
 y_tr = y_train.iloc[result[0]].reset_index(drop=True)
 X_val = X_train.iloc[result[1]].reset_index(drop=True)
 X_val.drop(labels='date_bin', axis=1, inplace=True)
 y_val = y_train.iloc[result[1]].reset_index(drop=True)
 np.random.seed(42*i)
 tf.keras.backend.clear_session()
tf.random.set_seed(42*i)
 batch_size = 8192
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_dnn(len(features), y_train.shape[1], hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 callbacks = get_callbacks(i)
 epochs = 200
history.append(clf.fit(X_tr, y_tr, epochs=epochs, batch_size=batch_size, validation_data=(X_val,y_val), callbacks=callbacks, verbose=0))
del clf, X_tr, y_tr, X_val, y_val, result
covid_pop_taken['State'].fillna("None", inplace=True)
 NOVAL = 'None'
covid_pop_taken['State'] = covid_pop_taken.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 covid_pop_taken.head(3 )<prepare_output>
env = janestreet.make_env()
df_train_copy2 = df_train.copy()
df_test_copy2 = df_test.copy()
df_test.head(3 )<merge>
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
<normalization><EOS> 
th = 0.501
clf0 = tf.keras.models.load_model("model-0.h5")
 clf2 = tf.keras.models.load_model("model-2.h5")
 clf4 = tf.keras.models.load_model("model-4.h5")
 models = [clf0, clf2, clf4]
test_df_columns = ['weight'] + [f'feature_{i}' for i in range(130)] + ['date']
index_features = [n for n,col in enumerate(test_df_columns)if col in features]
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].values[0] > 0:
x_tt = test_df.values[0][index_features].reshape(1,-1)
 x_tt[:, 1:] = fillna_npwhere_njit(x_tt[:, 1:][0], f_mean)
 pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0))
pred_df.action = int(pred >= th)
 else:
pred_df.action = 0
env.predict(pred_df )