Kaggle competition: Jane Street Market Prediction
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def loss_fn(ypred, label):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def seed_all(seed):
random.seed(seed)
 torch.manual_seed(seed)
 np.random.seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed )<set_options>
import random
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from tqdm import tqdm
from random import choices
from sklearn.model_selection import train_test_split
import lightgbm as lgbm
import pickle
warnings.filterwarnings("ignore")
 class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }<choose_model_class>
def load_pickle(fich):
with open(fich,"rb")as f :
contenu = pickle.load(f)
 return contenu
mx = BertStyleModel('bert')
 def _run() :
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs, targets)
 if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={loss}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
def eval_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 return fin_outputs, fin_targets
df_train = pd.read_csv('train.csv')
 df_valid = pd.read_csv('valid.csv')
 MAX_LEN = 192
TRAIN_BATCH_SIZE = 4
EPOCHS = 3
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 train_targets = df_train.target.values
valid_targets = df_valid.target.values
train_dataset = BERTDatasetTraining(
 comment_text=df_train.text.values,
targets=train_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=1 
)
 valid_dataset = BERTDatasetTraining(
 comment_text=df_valid.text.values,
targets=valid_targets,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=16,
sampler=valid_sampler,
drop_last=False,
num_workers=1 
)
 device = xm.xla_device()
model = mx.to(device)
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
lr = 0.4 * 1e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 xm.save(model.state_dict() , "bert.bin")
 np.save('bert', np.array(o))
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 xm.master_print(f'AUC = {auc}' )<train_model>
path = ".. /input/eda-missing-values-tsne-clustering-class-imb"
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = _run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )<string_transform>
SEED = 1111
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)
 }
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('bert')
 model.load_state_dict(torch.load("bert.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result.npy', ypred)
 return ypred 
<train_model>
random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )<load_from_csv>
print('load train start')
 try:
train = pd.read_parquet('train.parquet')
 except:
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv', index_col='ts_id')
 train.to_parquet('train.parquet')
 print('load train end' )
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result.npy')
 submit['target'] = ypred
submit['target'].hist()
plt.show()<save_to_csv>
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = load_pickle(os.path.join(path,"features_names"))
f_mean = load_pickle(os.path.join(path,"features_imputation"))
with open("imputation","wb")as f :
pickle.dump(f_mean,f)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, features]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit.csv', index=False)
<string_transform>
params={"num_leaves":300,
"max_bin":450,
"feature_fraction":0.52,
"bagging_fraction":0.52,
"objective":"binary",
"learning_rate":0.05,
"boosting_type":"gbdt",
"metric":"auc"
}
models = []
for i in tqdm(range(y_train.shape[1])) :
xtr,xval,ytr,yval = train_test_split(X_train ,y_train[:,i],test_size=0.2,stratify=y_train[:,i])
 d_train = lgbm.Dataset(xtr,label=ytr)
 d_eval = lgbm.Dataset(xval,label=yval,reference=d_train)
 clf = lgbm.train(params,d_train,valid_sets=[d_train,d_eval],num_boost_round=1500,\
early_stopping_rounds=50,verbose_eval=50)
 nom_fich = "weights_target_" + resp_cols[i]
clf.save_model(nom_fich)
 models.append(clf )
class BERTDatasetTraining:
def __init__(self, comment_text, targets, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
self.targets = targets
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.targets[item], dtype=torch.float)
 }<choose_model_class>
params={"num_leaves":300,
"max_bin":450,
"feature_fraction":0.52,
"bagging_fraction":0.52,
"objective":"binary",
"learning_rate":0.05,
"boosting_type":"gbdt",
"metric":"auc"
}
sing_model = []
for i in range(3):
clf = lgbm.Booster(model_file=os.path.join(path,"weights_{}".format(i)))
 sing_model.append(clf )
<train_model><EOS> 
f = np.median
th = 0.5000
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean
pred1 = f(np.stack([model.predict(x_tt)for model in models]),axis=0 ).T
pred2 = np.mean(np.stack([model.predict(x_tt)for model in sing_model]),axis=0)
 pred = 0.5 * pred1 + 0.5 * pred2
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )