Kaggle competition: Jane Street Market Prediction
def f1_smart(y_true, y_pred):
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_true,(y_pred > thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
best_f1 = thresholds[0][1]
return best_f1, best_thresh
def f1_smart_torch(y_true, y_pred):
y_true = y_true.cpu().data.numpy()
return f1_smart(y_true, y_pred)
 def save(m, info):
torch.save(info, 'best_model.info')
 torch.save(m, 'best_model.m')
 def load() :
m = torch.load('best_model.m')
 info = torch.load('best_model.info')
 return m, info<define_variables>
pca_components = 60
train_preds = np.zeros(( len(train_df)))
 test_preds = np.zeros(( len(test_df)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 best_thresholds = []
best_f1 = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 base_lr, max_lr = 0.0001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=max_lr)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 min_val_loss = 1000000000
step_size=300
best_thresh = 0
max_f1 = 0
for epoch in range(n_epochs):
start_time = time.time()
scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 model.train()
avg_loss = 0.
 step = 0
for x_batch, y_batch in tqdm(train_loader, disable=True):
if scheduler:
scheduler.batch_step()
step += 1
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_df)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
val_f1, threshold = f1_smart_torch(y_val_fold, valid_preds_fold)
 print('Optimal F1: {} at threshold: {}'.format(val_f1, threshold))
if avg_val_loss < min_val_loss:
print("Saving model with minimum Loss")
 save(m=model, info={'epoch': epoch, 'val_loss': avg_val_loss, 'val_f1': val_f1})
 min_val_loss = avg_val_loss
best_thresh = threshold
max_f1 = val_f1
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
best_thresholds.append(best_thresh)
 best_f1.append(max_f1)
 model,_ = load()
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
e_size = 64
fc_input = pca_components
h_dims = [512,512,256,128]
dropout_rate = 0.5
epochs = 200
minibatch_size = 100000
class MarketPredictor(nn.Module):
def __init__(self):
super(MarketPredictor, self ).__init__()
self.e = nn.Embedding(2,e_size)
 self.deep = nn.Sequential(
 nn.Linear(fc_input,h_dims[0]),
nn.BatchNorm1d(h_dims[0]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[0],h_dims[1]),
nn.BatchNorm1d(h_dims[1]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[1],h_dims[2]),
nn.BatchNorm1d(h_dims[2]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[2],h_dims[3]),
nn.BatchNorm1d(h_dims[3]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[3],e_size),
nn.BatchNorm1d(e_size),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate)
)
 self.reduce = nn.utils.weight_norm(nn.Linear(e_size,1))
self.sig = nn.Sigmoid()
def forward(self,xi,xf):
e_out = self.e(xi)
 f_out = self.deep(xf)
 ef_out = self.reduce(e_out+f_out)
 sig_out = self.sig(ef_out)
 return sig_out
search_result = threshold_search(y_train, train_preds)
 search_result<save_to_csv>
epochs = 200
path = '/kaggle/input/pytorch-nn-model/marketpredictor_state_dict_'+str(epochs)+'epochs.pt'
model = MarketPredictor()
model.load_state_dict(torch.load(path,map_location=dev))
model.to(dev)
 model.eval()
submission = test_df[['qid']].copy()
submission['prediction'] = test_preds > search_result['threshold']
submission.to_csv('submission.csv', index=False )<string_transform>
with open('/kaggle/input/pytorch-nn-model/feature_processing.pkl', 'rb')as f:
sc, pca, maxindex, fill_val = pickle.load(f )
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('spliting data')
 df_train,df_test = train_test_split(df, random_state=1)
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df_train['question_text'])
 x_test = tokenizer.texts_to_sequences(df_test['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 x_test = pad_sequences(x_test, maxlen=max_len)
 y_train = df_train['target'].values
y_test = df_test['target'].values
print(x_train.shape)
 print(y_train.shape)
<compute_train_metric>
feature_names = ['feature_'+str(i)for i in range(1,130)]
exclude = np.where([maxindex[i,1] > 100 and maxindex [i,2] > 1 for i in range(129)])[0]
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
env = janestreet.make_env()
iter_test = env.iter_test()
<statistical_test><EOS> 
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() == 0:
sample_prediction_df.action = 0
else:
test_df_features = test_df[feature_names].to_numpy()
for i in exclude:
if test_df_features[0,i] == maxindex[i,0]:
test_df_features[0,i] = fill_val[i]
test_df_int_features = test_df['feature_0'].to_numpy()
nans = np.isnan(test_df_features)
 for i in range(129):
if nans[0,i]:
test_df_features[0,i] = fill_val[i]
test_df_features_scaled = sc.transform(test_df_features)
 test_df_features_pca=pca.transform(test_df_features_scaled)
 itensor = torch.tensor(( test_df_int_features+1)//2,dtype=torch.long,device=dev)
 ftensor = torch.tensor(test_df_features_pca,dtype=torch.float,device=dev)
 s = model(itensor,ftensor ).item()
sample_prediction_df.action = int(np.round(s))
env.predict(sample_prediction_df)