Kaggle competition: Jane Street Market Prediction
Italy_S = Italy_SIR_df['S']
Italy_I = Italy_SIR_df['I']
Italy_R = Italy_SIR_df['R']
Italy_s = np.array(Italy_S, dtype=float)
 Italy_i = np.array(Italy_I, dtype=float)
 Italy_r = np.array(Italy_R, dtype=float)
 display(Italy_s[9],Italy_i[9], Italy_r[9] )<train_model>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 DATA_PATH = '.. /input/jane-street-market-prediction/'
NFOLDS = 5
TRAIN = False
CACHE_PATH = '.. /input/mlp012003weights'
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
feat_cols = [f'feature_{i}' for i in range(130)]
target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')
 all_feat_cols = [col for col in feat_cols]
all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])
 class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.2)
 dropout_rate = 0.2
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.LeakyReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.LeakyReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.LeakyReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.LeakyReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x
if True:
device = torch.device("cpu")
 model_list = []
tmp = np.zeros(len(feat_cols))
for _fold in range(NFOLDS):
torch.cuda.empty_cache()
model = Model()
model.to(device)
 model_weights = f"{CACHE_PATH}/online_model{_fold}.pth"
model.load_state_dict(torch.load(model_weights, map_location=torch.device('cpu')))
 model.eval()
model_list.append(model )
def SIR_model(y,t,beta,gamma):
S, I, R = y
N = 60480000
dS_dt = -1*beta*I*S/N
dI_dt =(beta*I*S/N)- gamma*I
dR_dt = gamma*I
return([dS_dt, dI_dt, dR_dt])
 def fit_odeint(x, beta, gamma):
return integrate.odeint(SIR_model,(S0, I0, R0), x, args=(beta, gamma)) [:,1]<train_model>
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.load_weights('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 tf_models = [clf]
xdata = Italy_SIR_df.Day_num
ydata = Italy_i
xdata = np.array(xdata, dtype=float)
 S0 = 60479998.0
I0 = 2.0
R0 = 0
y = S0, I0, R0
popt, pcov = optimize.curve_fit(fit_odeint, xdata, ydata)
 fitted = fit_odeint(xdata, *popt )<load_from_csv>
N_FEAT_TAGS = 29
DEVICE = device
N_FEATURES = 130
THREE_HIDDEN_LAYERS = [400, 400, 400]
class FFN(nn.Module):
def __init__(self, inputCount=130, outputCount=5, hiddenLayerCounts=[150, 150, 150],
drop_prob=0.2, nonlin=nn.SiLU() , isOpAct=False):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.batchnorm0 = nn.BatchNorm1d(inputCount)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if isOpAct:
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X):
X = self.batchnorm0(X)
 X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outDense:
X = self.outDense(X)
 if self.outActivtn:
X = self.outActivtn(X)
 return X
class Emb_NN_Model(nn.Module):
def __init__(self, three_hidden_layers=THREE_HIDDEN_LAYERS, embed_dim=(N_FEAT_TAGS), csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Emb_NN_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 drop_prob = 0.5
self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(three_hidden_layers[0]+embed_dim),(three_hidden_layers[1]+embed_dim),(three_hidden_layers[2]+embed_dim)], drop_prob=drop_prob)
 self.outDense = nn.Linear(three_hidden_layers[2]+embed_dim, 5)
 return
def features2emb(self):

 all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS))
tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, cat_featrs, features):

 cat_featrs = None
features = features.view(-1, N_FEATURES)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 return out_logits
comp_df = pd.read_csv('.. /input/corona-virus-report/covid_19_clean_complete.csv')
 comp_df.head(5 )<correct_missing_values>
embNN_model = Emb_NN_Model()
try:
embNN_model.load_state_dict(torch.load(".. /input/janefastai-featinteraction-embeddingnn5-300x3/Jane_EmbNN5_auc_400_400_400.pth"))
except:
embNN_model.load_state_dict(torch.load(".. /input/janefastai-featinteraction-embeddingnn5-300x3/Jane_EmbNN5_auc_400_400_400.pth", map_location='cpu'))
embNN_model = embNN_model.eval()
comp_df['Province/State'].fillna("None", inplace=True)
 comp_df[comp_df.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()<data_type_conversions>
env = janestreet.make_env()
env_iter = env.iter_test()
comp_df['Date'] = pd.to_datetime(comp_df['Date'] )<merge>
if True:
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, feat_cols].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean
cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]
cross_1_2 = x_tt[:, 1] /(x_tt[:, 2] + 1e-5)
 feature_inp = np.concatenate((
x_tt,
np.array(cross_41_42_43 ).reshape(x_tt.shape[0], 1),
np.array(cross_1_2 ).reshape(x_tt.shape[0], 1), 
), axis=1)
 torch_pred = np.zeros(( 1, len(target_cols)))
 for model in model_list:
torch_pred += model(torch.tensor(feature_inp, dtype=torch.float ).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS
torch_pred = np.median(torch_pred)
 tf_pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in tf_models],axis=0))
x_tt = torch.tensor(x_tt ).float().view(-1, 130)
 embnn_p = np.median(torch.sigmoid(embNN_model(None, x_tt)).detach().cpu().numpy().reshape(( -1, 5)) , axis=1)
 pred_pr = torch_pred*0.42 + tf_pred*0.42 + embnn_p*0.16
pred_df.action = np.where(pred_pr >= 0.4978, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )
<create_dataframe><EOS> 
print("Done !" )