Kaggle competition: Jane Street Market Prediction
%%time
torch_bi_lstm = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
bidirectional=True,
hidden_dim=50,
max_iter=50,
eta=0.05)
 _ = torch_bi_lstm.fit(X_train, y_train )<predict_on_test>
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
%%time
predictions = torch_bi_lstm.predict(X_dev)
 print(classification_report(y_dev, predictions)) 
<train_model>
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c]
%%time
torch_bilstm_attn_manning2015 = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="GlobalAttnManning2015",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attn_manning2015.fit(X_train, y_train )<predict_on_test>
print('Train Nan Valued colas: %d' %train.isna().any().sum() )
%%time
predictions = torch_bilstm_attn_manning2015.predict(X_dev)
 print(classification_report(y_dev, predictions)) 
<train_model>
f_mean = train[features[1:]].mean()
train = train.query('weight > 0' ).reset_index(drop = True)
%%time
torch_bilstm_attn_ShouPeng2016 = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="AttnShouPeng2016",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attn_ShouPeng2016.fit(X_train, y_train )<predict_on_test>
train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 np.save('f_mean.npy', f_mean )
%%time
predictions = torch_bilstm_attn_ShouPeng2016.predict(X_dev)
 print(classification_report(y_dev, predictions)) 
<train_model>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model
class BertClassifierPredictor(BaseEstimator, ClassifierMixin):
def __init__(self, model=ClassificationModel('roberta', 'roberta-base', args={"overwrite_output_dir": True},
use_cuda=False)) :
self.model = model
logging.basicConfig(level=logging.INFO)
 transformers_logger = logging.getLogger("transformers")
 transformers_logger.setLevel(logging.WARNING)
 def fit(self, X, y):
X = X.tolist()
y = y.tolist()
self.classes_ = unique_labels(y)
 self.X_ = X
self.y_ = y
d = {'text': X, 'labels': y}
df = pd.DataFrame(data=d)
 self.model.train_model(df)
 return self
def predict(self, X):
return self.model.predict(X.tolist())[0]<choose_model_class>
batch_size = 4096
hidden_units = [384, 896, 896, 394]
dropout_rates = [
0.10143786981358652,
0.19720339053599725,
0.2703017847244654,
0.23148340929571917,
0.2357768967777311,
]
label_smoothing = 1e-2
learning_rate = 1e-7
oof = np.zeros(len(train['action']))
gkf = GroupKFold(n_splits = 5)
 for fold,(tr, te)in enumerate(gkf.split(train['action'].values, train['action'].values, train['date'].values)) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
ckp_path = f'JSModel_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 7, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr, validation_data =(X_val, y_val), epochs = 1000,
batch_size = batch_size, callbacks = [rlr, ckp, es], verbose = 0)
 oof[te] += model.predict(X_val, batch_size = batch_size * 4 ).ravel()
score = roc_auc_score(y_val, oof[te])
 print(f'Fold {fold} ROC AUC:\t', score)
 model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate / 100)
 model.load_weights(ckp_path)
 model.fit(X_val, y_val, epochs = 4, batch_size = batch_size, verbose = 0)
 model.save_weights(ckp_path )
%%time
os.write(1, b"Starting experiments w BERT finetuning
")
 model = ClassificationModel('bert', '/kaggle/input/simpletransformer-outputs/checkpoint-bert-epoch-1', args={"overwrite_output_dir": True}, use_cuda=False)
 clf = BertClassifierPredictor(model=model)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', clf)])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))<choose_model_class>
score_oof = roc_auc_score(train['action'].values, oof)
 print(score_oof )
%%time
logging.basicConfig(level=logging.INFO)
 transformers_logger = logging.getLogger("transformers")
 transformers_logger.setLevel(logging.WARNING)
 model = ClassificationModel('roberta', '/kaggle/input/simpletransformer-outputs/checkpoint-roberta-epoch-1', args={"overwrite_output_dir": True}, use_cuda=False)
 clf = BertClassifierPredictor(model=model)
 pipe_roberta_finetuned = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', clf)])
 predictions = pipe_roberta_finetuned.predict(dev_df['text'])
 print(classification_report(dev_df['target'], predictions)) 
<load_pretrained>
num_models = 2
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'./JSModel_{i}.hdf5')
 models.append(clf)
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer(model=model, tokenizer=tokenizer)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))<prepare_x_and_y>
f_mean = np.load('./f_mean.npy')
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 bertTransformer = BertTransformer(model=model, tokenizer=tokenizer, combine_sentence_tokens=False)
 X_train = bertTransformer.transform(train_df["text"])
 X_dev = bertTransformer.transform(dev_df["text"])
 y_train = train_df["target"].tolist()
y_dev = dev_df["target"].tolist() 
<train_model>
env = janestreet.make_env()
env_iter = env.iter_test()
%%time
torch_bilstm_attention_roberta_exp = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="AttnShouPeng2016",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attention_roberta_exp.fit(X_train, y_train )<predict_on_test>
opt_th = 0.502
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = 0.
 for clf in models:
pred += clf(x_tt, training = False ).numpy().item() / num_models
pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )