Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<load_pretrained>
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
import datatable as dt
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import kerastuner as kt
from sklearn.model_selection import KFold
from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples
from sklearn.utils.validation import _deprecate_positional_args
class BertStyleModel(torch.nn.Module):
def __init__(self, model_type):
super().__init__()
self.model_type = model_type
if(model_type == 'roberta'):
config_path = 'roberta-base'
model_path = 'roberta-base'
config = RobertaConfig.from_pretrained(config_path)
 config.output_hidden_states = True
self.bert = RobertaModel.from_pretrained(model_path, config=config)
 elif(model_type == 'distilbert'):
config_path = 'distilbert-base-uncased'
config = DistilBertConfig.from_json_file(config_path)
 model_path = 'distilbert-base-uncased'
config.output_hidden_states = True
self.bert = DistilBertModel.from_pretrained(model_path, config=config)
 elif(model_type == 'bert'):
config_path = 'bert-base-uncased'
config = BertConfig.from_pretrained(config_path)
 config.output_hidden_states = True
model_path = 'bert-base-uncased'
self.bert = BertModel.from_pretrained(model_path, config=config)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.cls_token_head = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.classifier = nn.Linear(768, 1)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):
if(self.model_type == 'roberta'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
elif(self.model_type == 'distilbert'):
outputs = self.bert(input_ids, attention_mask=attention_mask)
 hidden_layers = outputs[1]
elif(self.model_type == 'bert'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
hidden_states_cls_embeddings = [x[:, 0] for x in hidden_layers[-4:]]
x = torch.cat(hidden_states_cls_embeddings, dim=-1)
 cls_output = self.cls_token_head(x)
 logits = torch.mean(torch.stack([
self.classifier(self.high_dropout(cls_output))
for _ in range(5)
 ], dim=0), dim=0)
 outputs = logits
return outputs<set_options>
from janestreet_utils import *
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def loss_fn(ypred, label):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def seed_all(seed):
random.seed(seed)
 torch.manual_seed(seed)
 np.random.seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed )<set_options>
%%time
TRAINING = False
USE_FINETUNE = True
FOLDS = 5
SEED = 42
train = dt.fread('.. /input/jane-street-market-prediction/train.csv' ).to_pandas()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
warnings.filterwarnings("ignore")
 class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }<choose_model_class>
%%time
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/jsmpdata-encoder/encoder.hdf5')
 encoder.trainable = False
mx = BertStyleModel('bert')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "bert.bin")
 np.save('bert', np.array(o))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )<train_model>
model_fn = lambda hp: create_model(hp,X.shape[-1],y.shape[-1],encoder)
 tuner = CVTuner(
 hypermodel=model_fn,
oracle=kt.oracles.BayesianOptimization(
 objective= kt.Objective('val_auc', direction='max'),
num_initial_points=4,
max_trials=20))
FOLDS = 5
SEED = 42
models = []
hp = pd.read_pickle(f'.. /input/jsmpdata-tune/best_hp_{SEED}.pkl')
 for f in range(FOLDS):
model = model_fn(hp)
 if USE_FINETUNE:
model.load_weights(f'.. /input/jsmpdata-tune/model_{SEED}_{f}_finetune.hdf5')
 else:
model.load_weights(f'.. /input/jsmpdata-tune/model_{SEED}_{f}.hdf5')
 models.append(model)
<string_transform><EOS> 
f = np.median
models = models[-2:]
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )