Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<predict_on_test>
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
print('Predicting labels for {:,} test sentences...'.format(len(tensor_input)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 true_labels.append(label_ids)
 print('DONE.' )<save_to_csv>
features1 = [f'feature_{i}' for i in range(71)]
features2 = [f'feature_{i}' for i in range(71, 130)]
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
df_submission['target'] = flat_predictions
df_submission.to_csv('submission.csv', index = False)
 df_submission.head()<choose_model_class>
X_train = [train.loc[:, features1].values,
train.loc[:, features2].values]
model_type = 'distilbert'
model_name = 'distilbert-base-uncased'
with_kfold = False
weight = [0.43, 0.57]
dataset = 'DATA2'
n_splits = 1
seed = 42
model_args = {'fp16': False,
'train_batch_size': 4,
'gradient_accumulation_steps': 2,
'do_lower_case': True,
'learning_rate': 1e-05,
'overwrite_output_dir': True,
'manual_seed': seed,
'num_train_epochs': 2}<install_modules>
assert len(features)== len(features1)+ len(features2 )
!pip install --upgrade transformers
!pip install simpletransformers<set_options>
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)} )
warnings.simplefilter('ignore')
 pd.set_option('max_rows', 100)
 pd.set_option('max_colwidth', 2000 )<set_options>
def create_resnet(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = tf.keras.layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(300, activation="mish"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(300, activation = "mish")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(300, "mish"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(300, "mish"),
],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(300, kernel_initializer='lecun_normal', activation='mish'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(300, kernel_initializer='lecun_normal', activation='mish'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=['AUC'])
 return model
clf = create_resnet(len(features1), len(features2), len(resp_cols), 1e-02)
 clf.summary()
<load_from_csv><EOS> 
tf.keras.backend.clear_session()
clf = create_resnet(len(features1), len(features2), len(resp_cols), 3e-04)
 clf.fit(X_train, y_train, epochs=250, batch_size=200000)
 clf.save('resnet.h5')
 models = []
models.append(clf)
 th = 0.50
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model([x_tt[:, :len(features1)], x_tt[:, len(features1):]], training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )