Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<feature_engineering>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 DATA_PATH = '.. /input/jane-street-market-prediction/'
NFOLDS = 5
TRAIN = False
CACHE_PATH = '.. /input/mlp012003weights'
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
feat_cols = [f'feature_{i}' for i in range(130)]
target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')
 all_feat_cols = [col for col in feat_cols]
all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])
 class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.2)
 dropout_rate = 0.2
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.LeakyReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.LeakyReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.LeakyReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.LeakyReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x
if True:
device = torch.device("cpu")
 model_list = []
tmp = np.zeros(len(feat_cols))
for _fold in range(NFOLDS):
torch.cuda.empty_cache()
model = Model()
model.to(device)
 model_weights = f"{CACHE_PATH}/online_model{_fold}.pth"
model.load_state_dict(torch.load(model_weights, map_location=torch.device('cpu')))
 model.eval()
model_list.append(model )
Country_Dates['Fatality_Days'] = Country_Dates['Fatality_Days'] / np.timedelta64(1, 'D')
 Country_Dates.head(5 )<data_type_conversions>
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates )(x)
 for i in range(3):
x = tf.keras.layers.Dense(hidden_units )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = 160
dropout_rates = 0.2
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.load_weights('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 tf_models = [clf]
cond1 = train.Date == '2020-04-12'
Country_Counts = train[['Country_Region','Date','Fatalities','ConfirmedCases']][(cond1)]
df_Country2 = pd.merge(Country_Dates, Country_Counts, how='left', on=['Country_Region'])
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = df_Country2['Fatality_Rate'].fillna(0)
 df_Country2 = df_Country2.replace([np.inf, -np.inf], np.nan)
 df_Country3 = df_Country2[df_Country2['Fatality_Rate'] > 0].groupby(['Country_Region'])[["Fatality_Rate"]].sum().nlargest(8,'Fatality_Rate')
 df_Country3 = df_Country3.reset_index()<groupby>
N_FEAT_TAGS = 29
DEVICE = device
N_FEATURES = 130
THREE_HIDDEN_LAYERS = [400, 400, 400]
class FFN(nn.Module):
def __init__(self, inputCount=130, outputCount=5, hiddenLayerCounts=[150, 150, 150],
drop_prob=0.2, nonlin=nn.SiLU() , isOpAct=False):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.batchnorm0 = nn.BatchNorm1d(inputCount)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if isOpAct:
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X):
X = self.batchnorm0(X)
 X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outDense:
X = self.outDense(X)
 if self.outActivtn:
X = self.outActivtn(X)
 return X
class Emb_NN_Model(nn.Module):
def __init__(self, three_hidden_layers=THREE_HIDDEN_LAYERS, embed_dim=(N_FEAT_TAGS),
csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Emb_NN_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 drop_prob = 0.5
self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0,
hiddenLayerCounts=[(three_hidden_layers[0]+embed_dim), 
(three_hidden_layers[1]+embed_dim), 
(three_hidden_layers[2]+embed_dim)],
drop_prob=drop_prob)
 self.outDense = nn.Linear(three_hidden_layers[2]+embed_dim, 5)
 return
def features2emb(self):
all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS))
tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, cat_featrs, features):
cat_featrs = None
features = features.view(-1, N_FEATURES)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 return out_logits
confirmed_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_China = confirmed_China.join(fatalities_China)
 confirmed_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Italy = confirmed_Italy.join(fatalities_Italy)
 confirmed_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Spain = confirmed_Spain.join(fatalities_Spain)
 confirmed_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Iran = confirmed_Iran.join(fatalities_Iran)
 confirmed_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_France = confirmed_France.join(fatalities_France)
 confirmed_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Netherlands = confirmed_Netherlands.join(fatalities_Netherlands)
 confirmed_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_UK = confirmed_UK.join(fatalities_UK)
 confirmed_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_USA = confirmed_USA.join(fatalities_USA)
<merge>
embNN_model = Emb_NN_Model()
try:
embNN_model.load_state_dict(torch.load(".. /input/jane-embnn5-auc-400-400-400/Jane_EmbNN5_auc_400_400_400.pth"))
except:
embNN_model.load_state_dict(torch.load(".. /input/jane-embnn5-auc-400-400-400/Jane_EmbNN5_auc_400_400_400.pth", map_location='cpu'))
embNN_model = embNN_model.eval()
df_pop_sel = df_pop[['Country(or dependency)','Population(2020)','Density(P/Km²)','Land Area(Km²)','Med.Age','Urban Pop %']]
df_pop_sel.columns = ['Country_Region', 'Population(2020)', 'Density(P/Km²)','Land Area(Km²)', 'Med.Age', 'Urban Pop %']
df_pop_sel['Urban Pop %'] = df_pop_sel['Urban Pop %'].str.rstrip('%')
 df_pop_sel.loc[df_pop_sel['Country_Region']=='United States', 'Country_Region'] = 'US'
df_country_pop = pd.merge(df_pop_sel, df_Country, how='inner', on=['Country_Region'])
 df_country_pop<feature_engineering>
env = janestreet.make_env()
env_iter = env.iter_test()
<merge><EOS> 
if True:
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, feat_cols].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean
cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]
cross_1_2 = x_tt[:, 1] /(x_tt[:, 2] + 1e-5)
 feature_inp = np.concatenate((
x_tt,
np.array(cross_41_42_43 ).reshape(x_tt.shape[0], 1),
np.array(cross_1_2 ).reshape(x_tt.shape[0], 1), 
), axis=1)
 torch_pred = np.zeros(( 1, len(target_cols)))
 for model in model_list:
torch_pred += model(torch.tensor(feature_inp, dtype=torch.float ).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS
torch_pred = np.median(torch_pred)
 tf_pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in tf_models],axis=0))
x_tt = torch.tensor(x_tt ).float().view(-1, 130)
 embnn_p = np.median(torch.sigmoid(embNN_model(None, x_tt)).detach().cpu().numpy().reshape(( -1, 5)) , axis=1)
 pred_pr = torch_pred*0.4 + tf_pred*0.4 + embnn_p*0.2
pred_df.action = np.where(pred_pr >= 0.5, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )