Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<save_to_csv>
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
results.to_csv('submission.csv', index=False)
 display(results.head() )<set_options>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 style.use('fivethirtyeight')
 sns.set(style='whitegrid',color_codes=True )<load_from_csv>
TRAINING = False
USE_FINETUNE = False
FOLDS = 4
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
def seed_everything(seed=999):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_label(diagnosis):
return ','.join([str(i)for i in range(diagnosis + 1)])
 def get_train_df(seed, num_zeros=4000):
val_preds_id = pd.read_csv('.. /input/bd-peter-and-lex-validation-set/val.csv')['id_code']
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 df_train['is_valid'] = False
df_train.id_code = '.. /input/aptos2019-blindness-detection/train_images/' + df_train.id_code + '.png'
df_train.columns = ['image_path', 'diagnosis', 'is_valid']
extra_training_df = pd.read_csv('.. /input/diabetic-retinopathy-resized/trainLabels.csv')
 extra_training_df['is_valid'] = False
extra_training_df.image = '.. /input/diabetic-retinopathy-resized/resized_train/resized_train/' + extra_training_df.image + '.jpeg'
extra_training_df.columns = ['image_path', 'diagnosis', 'is_valid']
test_labels_15_df = pd.read_csv('.. /input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')
 del test_labels_15_df['Usage']
test_labels_15_df.columns = ['image_id', 'diagnosis']
test_labels_15_df['dataset_id'] = 'test_labels_15'
test_labels_15_df['image_path'] = '.. /input/resized-2015-2019-blindness-detection-images/resized test 15/' + test_labels_15_df.image_id + '.jpg'
test_labels_15_df['is_valid'] = True
test_labels_15_df = test_labels_15_df[['image_path', 'diagnosis', 'is_valid']]
df_train = pd.concat([
df_train,
extra_training_df[(extra_training_df.diagnosis == 0)&(extra_training_df.is_valid)],
extra_training_df[(extra_training_df.diagnosis == 0)& ~(extra_training_df.is_valid)].sample(n=num_zeros, random_state=seed),
extra_training_df[extra_training_df.diagnosis == 1],
extra_training_df[extra_training_df.diagnosis == 2],
extra_training_df[extra_training_df.diagnosis == 3],
extra_training_df[extra_training_df.diagnosis == 4],
pd.concat([
test_labels_15_df[test_labels_15_df.diagnosis == 0].sample(n=7900, random_state=420),
test_labels_15_df[test_labels_15_df.diagnosis != 0]
] ).sample(n=10_000, random_state=420),
] ).sample(frac=1, random_state=seed)
 df_train['label'] = df_train.diagnosis.apply(get_label)
 return df_train
def make_or_preds(model_name, learner, model_path, expected_val):
learn.load(model_path);
val_items = learn.data.valid_dl.dataset.items
val_preds, val_y = learn.get_preds(ds_type=DatasetType.Valid)
 metric = cohen_kappa_score(val_y.argmax(1 ).numpy() , get_output_preds(( val_preds > 0.5 ).numpy()), weights='quadratic')
 raw_preds = pd.DataFrame(val_preds.numpy())
 raw_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
val_preds_df = pd.concat([
pd.DataFrame({
'id_code': [v.split('/')[-1].split('.')[0] for v in val_items],
'diagnosis': val_y.argmax(1 ).numpy() ,
'preds': get_output_preds(( val_preds > 0.5 ).numpy())
 }),
raw_preds
], axis=1)
 val_preds_df.to_csv(f'{model_name}_val_preds.csv', index=False)
 test_items = learn.data.test_dl.dataset.items
test_preds, __ = learn.get_preds(ds_type=DatasetType.Test)
 raw_test_preds = pd.DataFrame(test_preds.numpy())
 raw_test_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
test_preds_df = pd.concat([
pd.DataFrame({
'id_code': [v.split('/')[-1].split('.')[0] for v in test_items],
'preds': get_output_preds(( test_preds > 0.5 ).numpy())
 }),
raw_test_preds
], axis=1)
 test_preds_df.to_csv(f'{model_name}_test_preds.csv', index=False)
 print(f'Val kappa score: {metric}(expected: {expected_val})')
 def avg_tta_score(model_name):
no_flip = pd.read_csv(f'{model_name}_val_preds.csv' ).sort_values('id_code')
 flip = pd.read_csv(f'{model_name}-flip_val_preds.csv' ).sort_values('id_code')
 val_preds_avg = no_flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5 + flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5
return cohen_kappa_score(flip.diagnosis, get_output_preds(( val_preds_avg > 0.5)) , weights='quadratic')
 class ConfusionMatrix(Callback):
"Computes the confusion matrix."
def on_train_begin(self, **kwargs):
self.n_classes = 0
def on_epoch_begin(self, **kwargs):
self.cm = None
def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
preds = torch.tensor(get_preds(( torch.sigmoid(last_output)> 0.5 ).cpu().numpy()))
targs = torch.tensor(get_preds(last_target.cpu().numpy()))
if self.n_classes == 0:
self.n_classes = last_output.shape[-1]
self.x = torch.arange(0, self.n_classes)
 cm =(( preds==self.x[:, None])&(targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)
 if self.cm is None: self.cm = cm
else: self.cm += cm
def on_epoch_end(self, **kwargs):
self.metric = self.cm
@dataclass
class KappaScore(ConfusionMatrix):
"Compute the rate of agreement(Cohens Kappa )."
weights:Optional[str]=None
def on_epoch_end(self, last_metrics, **kwargs):
sum0 = self.cm.sum(dim=0)
 sum1 = self.cm.sum(dim=1)
 expected = torch.einsum('i,j->ij',(sum0, sum1)) / sum0.sum()
if self.weights is None:
w = torch.ones(( self.n_classes, self.n_classes))
w[self.x, self.x] = 0
elif self.weights == "linear" or self.weights == "quadratic":
w = torch.zeros(( self.n_classes, self.n_classes))
w += torch.arange(self.n_classes, dtype=torch.float)
 w = torch.abs(w - torch.t(w)) if self.weights == "linear" else(w - torch.t(w)) ** 2
else: raise ValueError('Unknown weights.Expected None, "linear", or "quadratic".')
 k = torch.sum(w * self.cm)/ torch.sum(w * expected)
 return add_metrics(last_metrics, 1-k)
 class FlattenedLoss() :
"Same as `func`, but flattens input and target."
def __init__(self, func, *args, axis:int=-1, floatify:bool=False, is_2d:bool=True, **kwargs):
self.func,self.axis,self.floatify,self.is_2d = func(*args,**kwargs),axis,floatify,is_2d
functools.update_wrapper(self, self.func)
 def __repr__(self): return f"FlattenedLoss of {self.func}"
@property
def reduction(self): return self.func.reduction
@reduction.setter
def reduction(self, v): self.func.reduction = v
def __call__(self, input:Tensor, target:Tensor, **kwargs)->Rank0Tensor:
input = input.transpose(self.axis,-1 ).contiguous()
target = target.transpose(self.axis,-1 ).contiguous()
if self.floatify: target = target.float()
target =(target * 0.9 + 0.05)
 target[:,0] = 1
input = input.view(-1,input.shape[-1])if self.is_2d else input.view(-1)
 return self.func.__call__(input, target.view(-1), **kwargs)
 def LabelSmoothBCEWithLogitsFlat(*args, axis:int=-1, floatify:bool=True, **kwargs):
"Same as `nn.BCEWithLogitsLoss`, but flattens input and target."
return FlattenedLoss(nn.BCEWithLogitsLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)
 class ReconstructFixMultiCategoryList(MultiCategoryList):
def reconstruct(self, t):
try:
return super().reconstruct(t)
 except Exception as e:
return FloatItem(np.log(t))<load_from_csv>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.005),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' )<choose_model_class>
def create_model(input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(0.13 )(x)
 hidden_units = [384, 896, 896, 394]
for idx, hidden_unit in enumerate(hidden_units):
x = Dense(hidden_unit )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.relu )(x)
 x = Dropout(0.25 )(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(0.0005),loss=BinaryCrossentropy(label_smoothing=0.05),metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = get_efficientnet('b3', True, '.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth' )<choose_model_class>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/janestreetmodels/encoder.hdf5')
 encoder.trainable = False
seed_everything(422)
 data = get_data(seed=437, size=256, bs=64)
 learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-256',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',
expected_val=0.7273654273342949
)<choose_model_class>
FOLDS = 4
SEED = 42
if TRAINING:
gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
for fold,(train_indices, test_indices)in enumerate(splits):
model = create_model(130, 5, encoder)
 X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=4096,callbacks=[EarlyStopping('val_auc',mode='max',patience=10,restore_best_weights=True)])
 model.save_weights(f'./model_{SEED}_{fold}.hdf5')
 model.compile(Adam(0.00001),loss='binary_crossentropy')
 model.fit(X_test,y_test,epochs=3,batch_size=4096)
 model.save_weights(f'./model_{SEED}_{fold}_finetune.hdf5')
 else:
models = []
for f in range(FOLDS):
model = create_model(130, 5, encoder)
 if USE_FINETUNE:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}_finetune.hdf5')
 else:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}.hdf5')
 models.append(model)
seed_everything(422)
 data = get_data(seed=437, size=256, bs=64, tfms=((),(flip_lr(p=1))))
learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-256-flip',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',
expected_val=0.7273654273342949
)<compute_test_metric>
if not TRAINING:
f = np.median
models = models[-2:]
env = janestreet.make_env()
th = 0.503
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )