Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<choose_model_class>
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model 
<choose_model_class>
tf.random.set_seed(42)
 features_columns = ["feature_%d" % i for i in range(130)]
columns_dtypes = {}
for column in features_columns:
columns_dtypes[column] = "float32"
columns_dtypes["resp_1"] = "float32"
columns_dtypes["resp_2"] = "float32"
columns_dtypes["resp_3"] = "float32"
columns_dtypes["resp_4"] = "float32"
columns_dtypes["resp"] = "float32"
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', dtype=columns_dtypes)
 features = [c for c in train.columns if "feature" in c]
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<load_from_csv>
f_mean = train[features].median()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features] = train[features].fillna(f_mean)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
np.save('f_mean.npy', f_mean )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<load_pretrained>
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
nltk.download('stopwords')
 nltk.download('punkt')
<choose_model_class>
params = {
'colsample_bytree': 0.4,
'learning_rate': 0.001,
'max_depth': 12,
'subsample': 0.8,
'seed': 42,
'tree_method': 'gpu_hist'
}
params["objective"] = 'binary:logistic'
params["eval_metric"] = 'logloss'
replace_by_space_re = re.compile('[/() {}\[\]\|@,;]')
 bad_symbols_re = re.compile('[^0-9a-z ]')
 links_re = re.compile(r'http\S+')
 Stopwords = set(stopwords.words('english'))
Stopwords.remove('no')
 Stopwords.remove('not')
 lemmatizer = nltk.stem.WordNetLemmatizer()<categorify>
y_tr = train['action'].values
xgb_path = 'xgb_best.model'
dtrain = xgb.DMatrix(X, label=y_tr)
 bst = xgb.train(params, dtrain, 100, [(dtrain, 'train')])
 bst.save_model(xgb_path )
def text_prepare(text):

 text = text.lower()
text = re.sub(replace_by_space_re," ",text)
 text = re.sub(bad_symbols_re, "",text)
 text = re.sub(links_re, "",text)
 word_tokens = word_tokenize(text)
 filtered_tokens=[]
for word in word_tokens:
if word not in Stopwords:
filtered_tokens.append(lemmatizer.lemmatize(word))
text = " ".join(word for word in filtered_tokens)
 return text<feature_engineering>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
batch_size = 5000
hidden_units = [160, 160, 160]
dropout_rates = [
0.25,
0.25,
0.25,
0.25
]
label_smoothing = 1e-2
learning_rate = 1e-3
num_models = 2
train["text"] = [text_prepare(x)for x in train["text"]]
test["text"] = [text_prepare(x)for x in test["text"]]<categorify>
model = create_mlp(X.shape[1], y.shape[1], hidden_units, dropout_rates, label_smoothing, learning_rate)
 ckp_path = 'best_val_AUC.hdf5'
model.fit(X, y, epochs = 200, batch_size = batch_size, verbose = 0)
 model.save_weights('best_val_AUC.hdf5' )
train_input = bert_encode(train["text"], tokenizer, max_len=160)
 test_input = bert_encode(test["text"], tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
xgb_models = []
xgb_model = treelite.Model.from_xgboost(bst)
 xgb_model.export_lib(toolchain='gcc', libpath='./xgb.so', params={'parallel_comp': 32}, verbose=True)
 print('finish')
 predictor = treelite_runtime.Predictor('./xgb.so', verbose=True)
 xgb_models.append(predictor )
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=5,
callbacks=[checkpoint],
batch_size=16
)<predict_on_test>
env = janestreet.make_env()
env_iter = env.iter_test()
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
opt_th = 0.5
f = np.median
f_mean = np.load('./f_mean.npy')
 for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:,:].sum()):
x_tt[:, :] = np.nan_to_num(x_tt[:, :])+ np.isnan(x_tt[:, :])* f_mean
pred = []
for xgb_ in xgb_models:
batch = treelite_runtime.Batch.from_npy2d(x_tt, rbegin=0, rend=1)
 pred.append(np.array(xgb_.predict(batch)).reshape(1,1))
for clf in models:
predict = np.mean(clf(x_tt, training = False ).numpy() , axis = 0)
 pred.append(f(predict))
pred_df.action = np.where(np.mean(pred)>= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )