Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<train_model>
import numpy as np
import pickle
import os
from sklearn.model_selection import GroupKFold
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import random
from torch.utils.data import Dataset, DataLoader
import gc
from sklearn.metrics import precision_score, roc_auc_score
import pickle
import pandas as pd
import datatable as dt
import matplotlib.pylab as plt
import time
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )<load_from_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result.npy')
 submit['target'] = ypred
submit['target'].hist()
plt.show()<save_to_csv>
def utility_score_numba(date, weight, resp, action):
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / len(Pi))
u = min(max(t, 0), 6)* np.sum(Pi)
 return u
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit.csv', index=False)
<string_transform>
def ensemble_structure(number_models, num_features, MODEL_ROOT):
if number_models == 1:
hidden_dims = [150]
number_of_dims = [num_features]
input_dims = [np.arange(num_features)]
elif os.path.isfile(MODEL_ROOT + "/" + 'hidden_dims.pkl'):
with open(MODEL_ROOT + "/" + 'hidden_dims.pkl', 'rb')as handle:
hidden_dims = pickle.load(handle)
 with open(MODEL_ROOT + "/" + 'number_of_dims.pkl', 'rb')as handle:
number_of_dims = pickle.load(handle)
 with open(MODEL_ROOT + "/" + 'input_dims.pkl', 'rb')as handle:
input_dims = pickle.load(handle)
 else:
hidden_dims = np.random.randint(96, 160, number_models)
 number_of_dims = np.random.randint(int(num_features*0.7), num_features, number_models)
 input_dims = []
for i in range(number_models):
input_dims.append(np.random.randint(0, num_features, number_of_dims[i]))
with open(MODEL_ROOT + "/" + 'hidden_dims.pkl', 'wb')as handle:
pickle.dump(hidden_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open(MODEL_ROOT + "/" + 'number_of_dims.pkl', 'wb')as handle:
pickle.dump(number_of_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open(MODEL_ROOT + "/" + 'input_dims.pkl', 'wb')as handle:
pickle.dump(input_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)
 return hidden_dims, number_of_dims, input_dims
class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }<choose_model_class>
def fold_selection(folds, kf=0, random = False,
date=450, Debug=False):
folds['fold']=int(0)
 if Debug==True:
val = folds.sample(n=int(folds.shape[0]/10))
folds.loc[val.index, 'fold'] = int(1)
 else:
if random==True:
val = folds.sample(n=int(folds.shape[0] / 10))
folds.loc[val.index, 'fold'] = int(1)
 elif kf>0:
group_kfold = GroupKFold(n_splits=kf)
 group = folds['date']
f = 1
for train_index, test_index in group_kfold.split(folds, folds['resp'], group):
folds.loc[test_index, 'fold'] = int(f)
 f = f+1
else:
folds.loc[folds['date']>date, 'fold'] = int(1)
 folds['fold'] = folds['fold'].astype(int)
 return folds
mx = BertStyleModel('roberta')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 print("%s: traning"%device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "roberta.bin")
 np.save('roberta', np.array(t))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )<train_model>
class simple_linear_layer(nn.Module):
def __init__(self, input_dim, out_dim):
super(simple_linear_layer, self ).__init__()
self.dense = nn.Linear(input_dim, out_dim)
 self.batch_norm = nn.BatchNorm1d(out_dim)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.dense(x)
 x = self.batch_norm(x)
 x = self.dropout(x)
 return x
class MLP_Model(nn.Module):
def __init__(self, input_dim, hidden_dim, out_dim=4):
super(MLP_Model, self ).__init__()
self.block1 = simple_linear_layer(input_dim, hidden_dim)
 self.block2 = simple_linear_layer(hidden_dim, hidden_dim)
 self.block3 = nn.Linear(int(hidden_dim), out_dim)
 def forward(self, x):
v = F.relu(self.block1(x))
v = F.relu(self.block2(v))
out = self.block3(v)
 return out
class ResidualBlock(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim, downsample=None):
super(ResidualBlock, self ).__init__()
self.fc1 = nn.Linear(input_dim, hidden_dim)
 self.bn1 = nn.BatchNorm1d(hidden_dim)
 self.relu1 = nn.ReLU()
self.dropout1 = nn.Dropout(0.2)
 self.fc2 = nn.Linear(hidden_dim, output_dim)
 self.bn2 = nn.BatchNorm1d(output_dim)
 self.relu2 = nn.ReLU()
self.dropout2 = nn.Dropout(0.2)
 self.downsample = downsample
def forward(self, x):
residual = x
out = self.fc1(x)
 out = self.bn1(out)
 out = self.relu1(out)
 out = self.dropout1(out)
 out1 = self.fc2(out)
 out1 = self.bn2(out1)
 out1 = self.relu2(out1)
 out1 = self.dropout2(out1)
 if self.downsample:
residual = self.downsample(residual)
 out1 = torch.cat([out1, residual], dim=1)
 return out1
class ResNet(nn.Module):
def __init__(self, input_dim, hidden_dim, block, num_classes=4):
super(ResNet, self ).__init__()
self.layer1 = self.make_layer(block, input_dim, hidden_dim, hidden_dim)
 self.layer2 = self.make_layer(block, 2*hidden_dim, hidden_dim, hidden_dim)
 self.out = nn.Linear(2*hidden_dim, num_classes)
 def make_layer(self, block, input_dim, hidden_dim, output_dim):
downsample = None
if(input_dim != output_dim):
downsample = nn.Sequential(
 nn.Linear(input_dim, output_dim))
layer=block(input_dim, hidden_dim, output_dim, downsample)
 return layer
def forward(self, x):
out = self.layer1(x)
 out = self.layer2(out)
 out = self.out(out)
 return out
class Ensemble_MoaModel(nn.Module):
def __init__(self, input_dims, number_of_dims, hidden_dims,
model_name, out_dim=4):
super(Ensemble_MoaModel, self ).__init__()
self.models = torch.nn.ModuleList()
self.input_dims = input_dims
self.model_name = model_name
for i in range(len(hidden_dims)) :
if self.model_name=='Simple_MLP':
self.models.append(MLP_Model(input_dim=number_of_dims[i],
hidden_dim=hidden_dims[i], out_dim=out_dim))
elif self.model_name=="ResNet":
self.models.append(ResNet(input_dim=number_of_dims[i],
hidden_dim=hidden_dims[i],
block=ResidualBlock,
num_classes=out_dim))
else:
print("Please check model name.There is no this model!!!")
 def forward(self, x):
out = []
for i in range(len(self.input_dims)) :
temp = self.models[i](x[:, self.input_dims[i]])
 out.append(temp.unsqueeze(0))
out = torch.cat(out, dim=0)
 out = out.permute(1, 0, 2)
 return out
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )<string_transform>
class JaneDataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x': torch.tensor(self.features[idx, :], dtype=torch.float),
'y': torch.tensor(self.targets[idx], dtype=torch.float),
}
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x': torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
}
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('roberta')
 model.load_state_dict(torch.load("roberta.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result2.npy', ypred)
 return ypred 
<train_model>
def train_fn(model, optimizer, scheduler,
loss_fn, dataloader, device):
model.train()
final_loss = 0
train_preds = []
train_y = []
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 train_y.append(targets.detach().cpu().numpy())
 outputs = model(inputs)
 loss = 0
for i in range(outputs.shape[1]):
loss = loss + loss_fn(outputs[:, i],
targets.to(device))
loss.backward()
optimizer.step()
final_loss += loss.item()
train_preds.append(np.mean(outputs.sigmoid().detach().cpu().numpy() , axis=1))
final_loss /= len(dataloader)
 train_preds = np.concatenate(train_preds)
 train_y = np.concatenate(train_y)
 auc = roc_auc_score(train_y, train_preds)
 return auc
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )<save_to_csv>
def valid_fn(model, df, dataloader, device,
scheduler=None, loss_fn=None):
model.eval()
final_loss = 0
valid_preds = []
val_y = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 val_y.append(targets.detach().cpu().numpy())
 outputs = model(inputs)
 if loss_fn==None:
valid_preds.append(np.mean(outputs.sigmoid().detach().cpu().numpy() , axis=1))
final_loss = 0
else:
loss = loss_fn(torch.mean(outputs, dim=1), targets.to(device))
final_loss += loss.item()
valid_preds.append(np.mean(outputs.sigmoid().detach().cpu().numpy() , axis=1))
final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 val_y = np.concatenate(val_y)
 actual = np.where(valid_preds>=0.5, 1, 0)
 score = utility_score_numba(df['date'].values, df['weight'].values,
df['resp'].values, actual[:, 0])
 auc = roc_auc_score(val_y, valid_preds)
 if scheduler!=None:
scheduler.step(auc)
 return auc, score, valid_preds
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result2.npy')
 submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False)
<concatenate>
def run_training(folds, target, feature_cols, target_cols,
input_dims, number_of_dims, hidden_dims,
model_name, BATCH_SIZE, LEARNING_RATE, WEIGHT_DECAY,
EPOCHS, EARLY_STOP, EARLY_STOPPING_STEPS, fold, seed, MODEL_ROOT):
seed_everything(seed)
 train_df = folds.reset_index(drop=True)
 valid_df = folds[folds['fold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target].values
train_dataset = JaneDataset(x_train, y_train)
 valid_dataset = JaneDataset(x_valid, y_valid)
 trainloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 if os.path.isfile(MODEL_ROOT + "/"+ f"FOLD_{fold}_{seed}.pth"):
model_new = torch.load(MODEL_ROOT + "/"+ f"FOLD_{fold}_{seed}.pth")
 model_new.to(DEVICE)
 model_new.eval()
valid_tpr, valid_score, valid_preds = valid_fn(model_new, valid_df, validloader, DEVICE, scheduler=None, loss_fn=None)
 oof = valid_preds
else:
model_new = Ensemble_MoaModel(input_dims, number_of_dims, hidden_dims, model_name, out_dim=len(target_cols))
print(model_new)
 model_new.to(DEVICE)
 optimizer = torch.optim.Adam(model_new.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='max', factor=0.1, patience=3, verbose=True)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_tpr = -np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model_new, optimizer, scheduler
, loss_fn, trainloader, DEVICE)
 valid_tpr, valid_score, valid_preds = valid_fn(model_new, valid_df, validloader, DEVICE, scheduler=scheduler, loss_fn=loss_fn)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_auc: {train_loss}, valid_auc: {valid_tpr}, valid_score: {valid_score}")
 if valid_tpr > best_tpr:
best_tpr = valid_tpr
oof = valid_preds
early_step = 0
torch.save(model_new, MODEL_ROOT + "/"+ f"FOLD_{fold}_{seed}.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
del train_loss, valid_tpr, valid_score
gc.collect()
model_new = torch.load(MODEL_ROOT + "/"+ f"FOLD_{fold}_{seed}.pth")
 model_new.to(DEVICE)
 return oof, valid_df
Try to concat 2 model.
 I really think: ROBERTA is better for this task.So try to multiply Roberta result with greatter 1 number.Get result better.
 Summary: roberta*1.3 and get top 15%<load_pretrained>
EPOCHS = 50
BATCH_SIZE = 5000
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
Date = 475
Debug = False
Random = True
kf=0
bert = np.load('result.npy')
 roberta= np.load('result2.npy' )<define_variables>
target_cols = ['resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4']
useless_cols = ['date', 'weight', 'ts_id']
target = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
<save_to_csv><EOS> 
inference = True
if inference==True:
model = torch.load('.. /input/prerained-model-with-50-epoch/ResNet_5/FOLD_1_10.pth')
 f_mean = np.load('.. /input/prerained-model-with-50-epoch/fmean.npy')
 with open('.. /input/prerained-model-with-50-epoch/feature_cols.pkl', 'rb')as handle:
feature_cols = pickle.load(handle)
 env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tst = test_df.loc[:, feature_cols].values
if np.isnan(x_tst[:, :].sum()):
x_tst[:, 1:] = np.nan_to_num(x_tst[:, 1:])+ np.isnan(x_tst[:, 1:])* f_mean
pred = np.zeros(( 1, len(target_cols)))
 pred = np.mean(model(torch.tensor(x_tst, dtype=torch.float ).to(DEVICE)).sigmoid().detach().cpu().numpy() , axis=1)
 pred = np.mean(pred, axis=1)
 pred_df.action = np.where(pred >= 0.5, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)
 else:
train = dt.fread('.. /input/jane-street-market-prediction/train.csv' ).to_pandas()
feature_cols = list(train.drop(useless_cols + target_cols, axis=1))
for action, col in zip(target, target_cols):
train[action] = np.where(train[col]>0, 1, 0)
 for col in feature_cols:
if train[col].isnull().sum() /train.shape[0]>0.05:
feature_cols.remove(col)
 with open('feature_cols.pkl', 'wb')as handle:
pickle.dump(feature_cols, handle, protocol=pickle.HIGHEST_PROTOCOL)
 train = train[train['weight'] != 0]
train = train.loc[train['date']>85]
train = train.reset_index(drop=True)
 train.fillna(train.mean() ,inplace=True)
 f_mean = np.mean(train[feature_cols[1:]].values,axis=0)
 np.save('fmean.npy', f_mean)
 list_number_of_models=[5]
model_names = ["ResNet"]
SEED = 10
num_features=len(feature_cols)
 num_targets=len(target_cols)
 weights_mod = [0.5, 0.5]
oof_ensemble = np.zeros(( int(train.shape[0]/5), len(target_cols)))
 for m, model_name in enumerate(model_names):
for w, number_of_models in enumerate(list_number_of_models):
MODEL_ROOT = "%s_%s" %(model_name, number_of_models)
 fold_name = 'fold.pkl'
if not os.path.exists(MODEL_ROOT):
os.makedirs(MODEL_ROOT)
 train = fold_selection(train, kf=kf, random=Random, date=Date, Debug=Debug)
 hidden_dims, number_of_dims, input_dims = ensemble_structure(number_of_models, num_features, MODEL_ROOT)
 if kf==0:
oof, val_df = run_training(folds=train, target=target, feature_cols=feature_cols,
target_cols=target_cols, input_dims=input_dims,
number_of_dims=number_of_dims, hidden_dims=hidden_dims,
model_name=model_name, BATCH_SIZE=BATCH_SIZE, LEARNING_RATE=LEARNING_RATE,
WEIGHT_DECAY=WEIGHT_DECAY, EPOCHS=EPOCHS, EARLY_STOP=EARLY_STOP,
EARLY_STOPPING_STEPS=EARLY_STOPPING_STEPS, fold=1, seed=SEED,
MODEL_ROOT=MODEL_ROOT)
 for th in range(490, 520):
action = np.where(oof>th/1000, 1, 0)
 print(model_name, th, 'score:', utility_score_numba(date=val_df['date'].values, weight=val_df['weight'].values,
resp=val_df['resp'].values, action=action[:, 0]))
else:
for fold in range(kf):
oof, val_df = run_training(folds=train, target=target, feature_cols=feature_cols,
target_cols=target_cols, input_dims=input_dims,
number_of_dims=number_of_dims, hidden_dims=hidden_dims,
model_name=model_name, BATCH_SIZE=BATCH_SIZE, LEARNING_RATE=LEARNING_RATE,
WEIGHT_DECAY=WEIGHT_DECAY, EPOCHS=EPOCHS, EARLY_STOP=EARLY_STOP,
EARLY_STOPPING_STEPS=EARLY_STOPPING_STEPS, fold=fold+1, seed=SEED,
MODEL_ROOT=MODEL_ROOT)
 for th in range(490, 520):
action = np.where(oof>th/1000, 1, 0)
 print(model_name, th, 'score:', utility_score_numba(date=val_df['date'].values, weight=val_df['weight'].values,
resp=val_df['resp'].values, action=action[:, 0]))