Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<string_transform>
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
import pathlib
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
class BERTDatasetTest:
def __init__(self, comment_text, tokenizer, max_length):
self.comment_text = comment_text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.comment_text)
 def __getitem__(self, item):
comment_text = str(self.comment_text[item])
 comment_text = " ".join(comment_text.split())
 inputs = self.tokenizer.encode_plus(
 comment_text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_length, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
padding_length = self.max_length - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
}
def test_loop_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(
 input_ids=ids,
attention_mask=mask, 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
fin_outputs.extend(outputs_np)
 return fin_outputs
def test_model() :
MAX_LEN = 192
device = xm.xla_device()
model = BertStyleModel('roberta')
 model.load_state_dict(torch.load("roberta.bin"))
model.to(device)
 df_test = pd.read_csv('data/test.csv')
 df_test = df_test.fillna('')
 df_test['text'] = df_test['keyword'] + ' ' + df_test['location'] + ' '+ df_test['text']
df_test = df_test[['text']]
tokenizer = transformers.RobertaTokenizer.from_pretrained("roberta-base", do_lower_case=True)
 test_dataset = BERTDatasetTest(
 comment_text=df_test.text.values,
tokenizer=tokenizer,
max_length=MAX_LEN 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
drop_last=False,
num_workers=1 
)
 para_loader = pl.ParallelLoader(test_loader, [device])
 ypred = test_loop_fn(para_loader.per_device_loader(device), model, device)
 ypred= np.array(ypred)
 np.save('result2.npy', ypred)
 return ypred 
<train_model>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = test_model()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork' )<save_to_csv>
CV_STRATEGY = 'PurgedGroupTimeSeriesSplit'
SEED = 2021
START_DATE = 86
FOLDS = 5
DEBUG = False
submit = pd.read_csv('data/sample_submission.csv')
 ypred = np.load('result2.npy')
 submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False)
<concatenate>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
Try to concat 2 model.
 I really think: ROBERTA is better for this task.So try to multiply Roberta result with greatter 1 number.Get result better.
 Summary: roberta*1.3 and get top 15%<load_pretrained>
class GroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
kf = model_selection.KFold(n_splits=self.n_splits, shuffle=self.shuffle, random_state=self.random_state)
 unique_ids = X[group].unique()
for fold,(tr_group_idx, va_group_idx)in enumerate(kf.split(unique_ids)) :
tr_group, va_group = unique_ids[tr_group_idx], unique_ids[va_group_idx]
train_idx = np.where(X[group].isin(tr_group)) [0]
val_idx = np.where(X[group].isin(va_group)) [0]
yield train_idx, val_idx
class StratifiedGroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
groups = X[group].values
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(self.n_splits)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(self.random_state ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(self.n_splits):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(self.n_splits):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_idx = [i for i, g in enumerate(groups)if g in train_groups]
test_idx = [i for i, g in enumerate(groups)if g in test_groups]
yield train_idx, test_idx
bert = np.load('result.npy')
 roberta= np.load('result2.npy' )<define_variables>
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
bert_ = np.array([int(b>0.5)for b in bert])
 roberta_ = np.array([int(b>0.5)for b in roberta])
<save_to_csv>
if DEBUG:
train = train.sample(10000, random_state=SEED)
 X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
submit = pd.read_csv('sample_submission.csv')
 ypred = roberta2
submit['target'] = ypred
plt.hist(ypred)
 plt.show()
pred = [int(i> 0.5)for i in ypred]
plt.hist(pred)
 plt.show()
submit['target'] = pred
submit.to_csv('submit2.csv', index=False )<save_to_csv>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
submit = pd.read_csv('/kaggle/input/nlpstarted/submit2(1 ).csv')
 submit.to_csv('submission.csv', index=False )<import_modules>
def create_model(n_features, n_labels, encoder, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,))
input_2 = encoder(input_1)
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(512, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1), name='l2_norm'),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, ], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model
import tensorflow as tf
from tensorflow import keras
import numpy as np
import pandas as pd
import tensorflow_hub as hub
from sklearn.model_selection import train_test_split 
<load_from_csv>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-resnet-with-autoencoder-train/encoder.hdf5')
 encoder.trainable = False
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
<split>
%%time
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_model(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-resnet-with-autoencoder-train/model_{SEED}_{fold}.hdf5'))
models.append(model )
<prepare_x_and_y><EOS> 
f = np.median
th = 0.501
model = models[-1]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = model(x_tt, training = False ).numpy()
pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )