Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<normalization>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<categorify>
plt.style.use('fivethirtyeight')
 y_ = Fore.YELLOW
r_ = Fore.RED
g_ = Fore.GREEN
b_ = Fore.BLUE
m_ = Fore.MAGENTA
c_ = Fore.CYAN
sr_ = Style.RESET_ALL
warnings.filterwarnings('ignore')
 output_notebook()
class BackHook(torch.nn.Module):
def __init__(self, hook):
super(BackHook, self ).__init__()
self._hook = hook
self.register_backward_hook(self._backward)
 def forward(self, *inp):
return inp
@staticmethod
def _backward(self, grad_in, grad_out):
self._hook()
return None
class WeightDrop(torch.nn.Module):

 def __init__(self, module, weights, dropout=0, variational=False):
super(WeightDrop, self ).__init__()
self.module = module
self.weights = weights
self.dropout = dropout
self.variational = variational
self._setup()
self.hooker = BackHook(lambda: self._backward())
 def _setup(self):
for name_w in self.weights:
w = getattr(self.module, name_w)
 self.register_parameter(name_w + '_raw', Parameter(w.data))
def _setweights(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 mask = raw_w.new_ones(( raw_w.size(0), 1))
mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True)
 w = mask.expand_as(raw_w)* raw_w
rnn_w = getattr(self.module, name_w)
 rnn_w.data.copy_(w)
 setattr(self, name_w + "_mask", mask)
 def _backward(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 rnn_w = getattr(self.module, name_w)
 raw_w.grad = rnn_w.grad * getattr(self, name_w + "_mask")
 def forward(self, *args):
self._setweights()
return self.module(*self.hooker(*args))<choose_model_class>
%%time
folder_path = '.. /input/jane-street-market-prediction/'
train_data = dt.fread(folder_path + 'train.csv' ).to_pandas()
features = dt.fread(folder_path + 'features.csv' ).to_pandas()
sample = dt.fread(folder_path + 'example_sample_submission.csv' ).to_pandas()
test_data = dt.fread(folder_path + 'example_test.csv' ).to_pandas()
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = WeightDrop(nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.gru = WeightDrop(nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.tag_lstm = nn.LSTM(maxlen, hidden_size, bidirectional=True, batch_first=True)
 self.tag_gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.linear = WeightDrop(nn.Linear(hidden_size*8+2, 32), ['weight'], dropout=0.1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(32, 1)
 def forward(self, x, y, z):
x_embedding = self.embedding(x)
 x_lstm, _ = self.lstm(x_embedding)
 x_gru, _ = self.gru(x_lstm)
 x_lstm_atten = self.lstm_attention(x_lstm)
 x_gru_atten = self.gru_attention(x_gru)
 x_max_pool, _ = torch.max(x_gru, 1)
 y_lstm, _ = self.tag_lstm(y.view(len(y), -1, maxlen))
y_gru, _ = self.tag_gru(y_lstm)
 y_max_pool_1, _ = torch.max(y_lstm, 1)
 y_max_pool_2, _ = torch.max(y_gru, 1)
 y_max_pool = torch.add(y_max_pool_1, y_max_pool_2)
 conc = torch.cat(( x_lstm_atten, x_gru_atten, x_max_pool, y_max_pool, z), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<split>
print("Number of features with null values:",np.sum(train_data.isna().sum() >0))
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))<compute_test_metric>
train_data.fillna(train_data.mean() ,inplace=True)
 print("Number of features with null values:",np.sum(train_data.isna().sum() >0))
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_train_metric>
print("Number of rows with weight zero:",np.sum(train_data.weight==0))
print("Percentage of rows with weight zero",np.sum(train_data.weight==0)/len(train_data))
def threshold_search(y_true, y_proba):
precision, recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
search_result = {'threshold': best_th , 'f1': best_score}
return search_result<prepare_x_and_y>
print("Number of rows with resp zero:",np.sum(train_data.resp==0))
print("Percentage of rows with resp zero",np.sum(train_data.resp==0)/len(train_data))
x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
x_tag_test_cuda = torch.tensor(test_tag_X, dtype=torch.float32 ).cuda()
features_test_cuda = torch.tensor(test_features, dtype=torch.float32 ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda, x_tag_test_cuda, features_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<prepare_x_and_y>
def utility_score(df,score=0.5,testing=False,true_percentage=0.8):
if testing:
df['action'] = 0
false_percentage = 1 - true_percentage
mask1 = df['resp'] >= 0
mask2 = df['resp'] < 0
df.loc[mask1,'action'] = score
change_idx = df[mask1].sample(frac=false_percentage ).index
df.loc[change_idx,'action'] = 0
change_idx = df[mask2].sample(frac=false_percentage ).index
df.loc[change_idx,'action'] = score
df['pj'] = df['action'] * df['resp'] * df['weight']
pi = df[['date','pj']].groupby('date' ).agg({'pj':['sum']})
 pi.columns = ['pi']
pi['pi_sq'] = pi['pi']**2
t = np.sum(pi['pi'])/np.sqrt(np.sum(pi['pi_sq'])) * np.sqrt(250/len(pi['pi']))
u = min(max(t,0),6)* np.sum(pi['pi'])
 return u
else:
df['pj'] = df['action'] * df['resp'] * df['weight']
pi = df[['date','pj']].groupby('date' ).agg({'pj':['sum']})
 pi.columns = ['pi']
pi['pi_sq'] = pi['pi']**2
t = np.sum(pi['pi'])/np.sqrt(np.sum(pi['pi_sq'])) * np.sqrt(250/len(pi['pi']))
u = min(max(t,0),6)* np.sum(pi['pi'])
 return u
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
x_tag_train_fold = torch.tensor(train_tag_X[train_idx], dtype=torch.float32 ).cuda()
feature_train_fold = torch.tensor(train_features[train_idx], dtype=torch.float32 ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
x_tag_val_fold = torch.tensor(train_tag_X[valid_idx], dtype=torch.float32 ).cuda()
feature_val_fold = torch.tensor(train_features[valid_idx], dtype=torch.float32 ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 train = torch.utils.data.TensorDataset(x_train_fold, x_tag_train_fold, feature_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, x_tag_val_fold, feature_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 best_f1 = 0.
 train_preds_tmp = np.zeros(( len(train_X)))
 test_preds_tmp = np.zeros(( len(test_X)))
 for epoch in range(train_epochs):
start_time = time.time()
if epoch < 4:
optimizer = torch.optim.Adam(model.parameters())
 else:
optimizer = torch.optim.Adam(model.parameters() , lr=0.0001)
 model.train()
avg_loss = 0.
 for x_batch, x_tag_batch, feature_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch, x_tag_batch, feature_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 valid_true_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, x_tag_batch, feature_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch, x_tag_batch, feature_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
valid_true_fold[i * batch_size:(i+1)* batch_size] = y_batch.cpu().numpy() [:, 0]
elapsed_time = time.time() - start_time
search_result = threshold_search(valid_true_fold, valid_preds_fold)
 if search_result['f1'] > best_f1:
train_preds_tmp[valid_idx] = valid_preds_fold
for i,(x_batch, x_tag_batch, feature_batch)in enumerate(test_loader):
y_pred = model(x_batch, x_tag_batch, feature_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds_tmp = test_preds_fold / len(splits)
 print('Epoch {}/{}\tloss={:.4f}\tval_loss={:.4f}\tthreshold={:.2f}\tval_f1={:.8f}\ttime={:.2f}s [impoved]'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))
best_f1 = search_result['f1']
else:
print('Epoch {}/{}\tloss={:.4f}\tval_loss={:.4f}\tthreshold={:.2f}\tval_f1={:.8f}\ttime={:.2f}s [not impoved]'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))
train_preds[valid_idx] += train_preds_tmp[valid_idx]
test_preds += test_preds_tmp<compute_test_metric>
train_data['action'] =0
train_data.loc[train_data['resp']>0.0,'action'] = 1
features = [f"feature_{x}" for x in range(130)]
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
temp = train_data.groupby('action')[[f"feature_{x}" for x in range(130)]].mean()
diff_mean =(temp.iloc[1] - temp.loc[0] ).abs()
highest_diff_in_mean = diff_mean.sort_values(ascending=False)[:10].index.tolist()
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False)
 grouped = sub.groupby("prediction" ).size()
print(grouped)
 print(grouped[1] / grouped[0] )<compute_test_metric>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
def find_word(word, x):
regex = r"\b{}\b".format(word)
 return True if re.search(regex, x)else False<split>
features = [f'feature_{x}' for x in range(130)]
target_feature = 'action'
train_mean = train_data.mean()
train_data.fillna(train_mean,inplace=True)
 train_data['action'] = 0
train_data.loc[train_data['resp']>0.0,'action'] = 1
dates = train_data['date'].to_numpy()
target = train_data[target_feature].to_numpy()
train_data = train_data[features].to_numpy()
for i,q in enumerate(test_df.question_text.iloc[:]):
if find_word("anal", q):
print(sub.prediction.iloc[i], " : ", q )<load_from_csv>
params = {
'objective': 'binary',
'metrics':['auc'],
}
nfolds=3
group_gap = 31
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('val len:',len(val))
print('test len:',len(test))<feature_engineering>
kfold = PurgedGroupTimeSeriesSplit(n_splits=nfolds,group_gap=group_gap)
 lgb_models = list()
for k ,(train_idx,valid_idx)in enumerate(kfold.split(train_data,target.ravel() ,dates.ravel())) :
lgb_train = lgb.Dataset(train_data[train_idx],target[train_idx].ravel())
 lgb_valid = lgb.Dataset(train_data[valid_idx],target[valid_idx].ravel())
 model = lgb.train(
 params,
lgb_train,
valid_sets = [lgb_train,lgb_valid],
num_boost_round = 10000,
verbose_eval = 50,
early_stopping_rounds = 10, 
)
 lgb_models.append(model )
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )<train_model>
predictions = np.zeros(test_data.shape[0])
 for model in lgb_models:
predictions += model.predict(test_data[features])
 predictions /= len(lgb_models)
 sns.distplot(predictions);
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss<set_options>
del train_data, target
gc.collect()
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False)
 val_loader = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
shuffle=False,
sort=False)
 test_loader = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
shuffle=False,
sort=False)
 class Sentiment(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = int(vocab_vectors.shape[1]/2)
 self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear2 = nn.Linear(self.hidden_dim*vocab_vectors.shape[1],1)
 self.dropout = nn.Dropout(0.2)
 self.attn = nn.Linear(self.hidden_dim * 2, 100)
 self.attn_combine = nn.Linear(self.hidden_dim * 2, self.hidden_dim)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 lstm_out, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
z = torch.cat(( e, lstm_out), 1)
 attn_weights = F.softmax(self.attn(z), dim=1)
 attn_applied = torch.bmm(attn_weights,
lstm_out)
 output = torch.cat(( e, attn_applied), 1)
 attn_combine_output = self.attn_combine(output)
 out = self.dropout(F.relu(attn_combine_output))
out = out.reshape(out.shape[0],out.shape[1]*out.shape[2])
 out = self.linear2(out)
 return out
model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,train_batch in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(train_batch.text.cuda() ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_true = train_batch.target.float().cuda()
y_numpy_true = train_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,val_batch in enumerate(list(val_loader),1):
y_pred = model(val_batch.text.cuda() ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_true = val_batch.target.float().cuda()
y_numpy_true = val_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )<load_pretrained>
env = janestreet.make_env()
iter_test = env.iter_test()
print(os.listdir())
 model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))<init_hyperparams>
%%time
for(test_df, sample_prediction_df)in iter_test:
prediction = 0
for model in lgb_models:
prediction += model.predict(test_df[features])[0]
prediction /= len(lgb_models)
 prediction = prediction > 0.5
sample_prediction_df.action = prediction.astype(int)
 env.predict(sample_prediction_df )
<load_from_csv><EOS> 
submission = pd.read_csv('./submission.csv')
 submission.head()