Kaggle competition: Jane Street Market Prediction
class BackHook(torch.nn.Module):
def __init__(self, hook):
super(BackHook, self ).__init__()
self._hook = hook
self.register_backward_hook(self._backward)
 def forward(self, *inp):
return inp
@staticmethod
def _backward(self, grad_in, grad_out):
self._hook()
return None
class WeightDrop(torch.nn.Module):

 def __init__(self, module, weights, dropout=0, variational=False):
super(WeightDrop, self ).__init__()
self.module = module
self.weights = weights
self.dropout = dropout
self.variational = variational
self._setup()
self.hooker = BackHook(lambda: self._backward())
 def _setup(self):
for name_w in self.weights:
print('Applying weight drop of {} to {}'.format(self.dropout, name_w))
w = getattr(self.module, name_w)
 self.register_parameter(name_w + '_raw', Parameter(w.data))
def _setweights(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 mask = raw_w.new_ones(( raw_w.size(0), 1))
mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True)
 w = mask.expand_as(raw_w)* raw_w
rnn_w = getattr(self.module, name_w)
 rnn_w.data.copy_(w)
 setattr(self, name_w + "_mask", mask)
 def _backward(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 rnn_w = getattr(self.module, name_w)
 raw_w.grad = rnn_w.grad * getattr(self, name_w + "_mask")
 def forward(self, *args):
self._setweights()
return self.module(*self.hooker(*args))<choose_model_class>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 DATA_PATH = '.. /input/jane-street-market-prediction/'
NFOLDS = 5
TRAIN = False
CACHE_PATH = '.. /input/mlp012003weights'
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
feat_cols = [f'feature_{i}' for i in range(130)]
target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')
 all_feat_cols = [col for col in feat_cols]
all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])
 class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.2)
 dropout_rate = 0.2
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.LeakyReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.LeakyReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.LeakyReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.LeakyReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x
if True:
device = torch.device("cpu")
 model_list = []
tmp = np.zeros(len(feat_cols))
for _fold in range(NFOLDS):
torch.cuda.empty_cache()
model = Model()
model.to(device)
 model_weights = f"{CACHE_PATH}/online_model{_fold}.pth"
model.load_state_dict(torch.load(model_weights, map_location=torch.device('cpu')))
 model.eval()
model_list.append(model )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = WeightDrop(nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.gru = WeightDrop(nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = WeightDrop(nn.Linear(hidden_size*8, 24), ['weight'], dropout=0.1)
 self.relu = nn.ReLU()
self.out = nn.Linear(24, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
out = self.out(conc)
 return out<prepare_x_and_y>
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.load_weights('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 tf_models = [clf]
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
class FFN(nn.Module):
def __init__(self, inputCount=160, outputCount=0, hiddenLayerCounts=[80, 40, 20], drop_prob=0.2, nonlin=nn.SiLU()):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X, **kwargs):
X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outActivtn:
X = self.outActivtn(self.outDense(X))
return X
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
N_FEAT_TAGS = 29
DEVICE = device
class Clf_Model(nn.Module):
def __init__(self, embed_dim=N_FEAT_TAGS, csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Clf_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(160+embed_dim),(100+embed_dim),(50+embed_dim)], drop_prob=0.25)
 self.outDense = nn.Linear(50+embed_dim, 5)
 self.criterion = nn.BCEWithLogitsLoss()
return
def features2emb(self):

 all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS))
tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, features, labels=None):

 features = features.view(-1, 130)
 if labels is not None:
labels = labels.view(-1, 5)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 if labels is None:
out_probs = F.sigmoid(out_logits)
 Probs = namedtuple('Probs',['logits'])
 out_probs = Probs(out_probs)
 return out_probs
batchLoss = self.criterion(out_logits, labels.float())
 out_probs = F.sigmoid(out_logits)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_probs)
 return loss_logits
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False)
 print(sub.groupby("prediction" ).size() )<set_options>
embNN_model = Clf_Model()
try:
embNN_model.load_state_dict(torch.load(".. /input/jane-emb-nn5-clf-end-noscal-160-100-50/jane_emb_nn5_clf_end_noscal_160_100_50.pt"))
except:
embNN_model.load_state_dict(torch.load(".. /input/jane-emb-nn5-clf-end-noscal-160-100-50/jane_emb_nn5_clf_end_noscal_160_100_50.pt", map_location='cpu'))
embNN_model = embNN_model.eval()
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )<load_from_csv>
env = janestreet.make_env()
env_iter = env.iter_test()
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('val len:',len(val))
print('test len:',len(test))<normalization>
if True:
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, feat_cols].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean
cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]
cross_1_2 = x_tt[:, 1] /(x_tt[:, 2] + 1e-5)
 feature_inp = np.concatenate((
x_tt,
np.array(cross_41_42_43 ).reshape(x_tt.shape[0], 1),
np.array(cross_1_2 ).reshape(x_tt.shape[0], 1), 
), axis=1)
 torch_pred = np.zeros(( 1, len(target_cols)))
 for model in model_list:
torch_pred += model(torch.tensor(feature_inp, dtype=torch.float ).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS
torch_pred = np.median(torch_pred)
 tf_pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in tf_models],axis=0))
x_tt = torch.tensor(x_tt ).float().view(-1, 130)
 embnn_p = np.median(embNN_model(x_tt ).logits.detach().cpu().numpy().reshape(( -1, 5)) , axis=1)
 pred_pr = torch_pred*0.35 + tf_pred*0.35 + embnn_p*0.30
pred_df.action = np.where(pred_pr >= 0.5, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300)
 print(glove.vectors.shape )<train_model>
print("Done !" )