Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<normalization>
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
encoder_out_dim=64
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.gru = nn.GRU(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(360, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_gru, _ = self.gru(h_embedding)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<split>
class Autoencoder(nn.Module):
def __init__(self):
super(Autoencoder, self ).__init__()
self.encoder = nn.Sequential(nn.BatchNorm1d(130),
LinBnDrop(130, 256, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(256, 512, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(512, 256, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(256, encoder_out_dim, bn=False, act=None, lin_first=False))
self.decoder = nn.Sequential(nn.BatchNorm1d(64),
LinBnDrop(64, 400, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(400, 800, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(800, 400, bn=True, p=0.12, act=Mish() , lin_first=False),
LinBnDrop(400, 130, bn=False, act=None, lin_first=False))
def encode(self, x):
return self.encoder(x)
 def decode(self, x):
return self.decoder(x)
 def forward(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED ).split(train_X, train_y))<compute_test_metric>
ae = Learner(None, Autoencoder() , loss_func=1)
 ae.load('/kaggle/input/jstreetae1/autoencoder_rmse' )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
class Model(nn.Module):
def __init__(self, input_dim):
super(Model, self ).__init__()
self.encoder = ae.model.encoder
self.encoder.eval()
self.layers = nn.Sequential(nn.BatchNorm1d(input_dim),
LinBnDrop(input_dim, 400, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(400, 800, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(800, 400, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(400, 5, bn=False, act=None, lin_first=False))
def forward(self, x):
encoded_x = self.encoder(x ).detach()
x = t.cat(( encoded_x, x), dim=1)
 x = self.layers(x)
 return x
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<categorify>
model_nn = Model(len(features)+ encoder_out_dim)
 model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/jstreetmlpae1/ae_roc' )
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )