Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<choose_model_class>
warnings.filterwarnings('ignore')
 tf.random.set_seed(41)
%%time
clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<choose_model_class>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('clf', clf)])
 parameters = [
{'clf__hidden_layer_sizes': [(100, 100),(100,),(100, 50),(50, 50)]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/mlp_clf_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/mlp_clf_gs_results.csv")
 grid_search_pd 
<choose_model_class>
def create_mlp_wrapper(train, params, filename, features):
batch_size = params['batch_size']
hidden_units = [params['hidden_unit_1'], params['hidden_unit_2'], params['hidden_unit_3'], params['hidden_unit_4']]
dropout_rates = [params['dropout_rate_1'], params['dropout_rate_2'], params['dropout_rate_3'], params['dropout_rate_4'], params['dropout_rate_5']]
label_smoothing = params['label_smoothing']
learning_rate = params['learning_rate']
time='all'
oof = []
gkf = GroupKFold(n_splits = 5)
 scores = []
utility_noisy = pd.DataFrame(columns=features,index=range(5))
scores_noisy = pd.DataFrame(columns=features,index=range(5))
for fold,(tr, te)in enumerate(gkf.split(train['resp'].values, train['resp'].values, train['date'].values)) :
X_tr, X_val = train.iloc[tr][features].values, train.iloc[te][features].values
y_tr, y_val = train.iloc[tr]['action'].values, train.iloc[te]['action'].values
ckp_path = f'JSModel_{filename}_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 7, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr, validation_data =(X_val, y_val), epochs = 1000,
batch_size = batch_size, callbacks = [rlr, ckp, es], verbose = 0)
 rng = np.random.default_rng()
for i, feature in enumerate(features):
gc.collect()
X_val_noisy = X_val.copy()
X_val_noisy[:,i] = rng.permutation(X_val_noisy[:,i])
 pred = model.predict(X_val_noisy, batch_size = batch_size * 4 ).ravel()
scores_noisy.loc[fold, feature] = roc_auc_score(y_val,(pred>.5 ).astype(int))
utility_noisy.loc[fold, feature] = utility_score(train.iloc[te]['date'].values, train.iloc[te]['weight'].values, train.iloc[te]['resp'],(pred>.5 ).astype(int))
oof.append(model.predict(X_val, batch_size = batch_size * 4 ).ravel())
 score = roc_auc_score(y_val, oof[fold])
 print(f'Fold {fold} ROC AUC:\t', round(score, 4))
K.clear_session()
del model
rubbish = gc.collect()
return oof, scores_noisy, utility_noisy
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('clf', clf)])
 parameters = [
{'clf__hidden_layer_sizes': [(100, 50, 50),(100,100, 50)]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/mlp_clf2_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/mlp_clf2_gs_results.csv")
 grid_search_pd 
<categorify>
def utility_score(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u
bertTransformer = BertTransformer()
bertTransformer.transform(["granola bars"] ).shape<choose_model_class>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features[1:]].mean()
train = train.query('weight > 0' ).reset_index(drop = True)
 train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 params = {'batch_size': 4096, 'dropout_rate_1': 0.08949964830076425, 'dropout_rate_2': 0.09214922878769727, 'dropout_rate_3': 0.10007316489415027, 'dropout_rate_4': 0.2416444299467454, 'dropout_rate_5': 0.1338760637313039, 'hidden_unit_1': 245, 'hidden_unit_2': 678, 'hidden_unit_3': 759, 'hidden_unit_4': 471, 'label_smoothing': 0.0007423928500131863, 'learning_rate': 0.0003772663214527269}
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('predictor', clf)])
 parameters = [
{'vectorizer__embedding_func': [sum_all_tokens, mean_across_all_tokens, concat_all_tokens, CLS_token_embedding]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/emb_funcs_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/emb_funcs_gs_results.csv")
 grid_search_pd 
<load_pretrained>
gc.collect()
oof_pred, scores_noisy, utility_noisy = create_mlp_wrapper(train, params, 'filename', features)
 gc.collect()
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 clf = XGBClassifier(n_estimators=300)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer(model=model, tokenizer=tokenizer)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))<prepare_x_and_y>
def mutate(x, p):
mutate_idx = np.random.choice(x.shape[0], int(x.shape[0]*p),replace=False)
 x[mutate_idx] = np.where(x[mutate_idx] == 1, 0, 1)
 return x
%%time
os.write(1, b"Starting experiments with sequential models
")
 bertTransformer = BertTransformer(combine_sentence_tokens=False)
 X_train = bertTransformer.transform(train_df["text"])
 X_dev = bertTransformer.transform(dev_df["text"])
 y_train = train_df["target"].tolist()
y_dev = dev_df["target"].tolist()<train_model>
gkf = GroupKFold(n_splits = 5)
 utilities = []
for fold,(tr, te)in enumerate(gkf.split(train['resp'].values, train['resp'].values, train['date'].values)) :
oof_pred_, oof_real = oof_pred[fold], train.iloc[te]['action'].values
fold_utilities = []
ps = np.linspace(0.0, 0.7, num=100)
 for p in ps:
mutated_pred = mutate(( oof_pred_>0.5 ).astype(int), p)
 fold_utilities.append(utility_score(train.iloc[te]['date'].values, train.iloc[te]['weight'].values, train.iloc[te]['resp'], mutated_pred))
utilities.append(fold_utilities )
%%time
torch_rnn = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
bidirectional=False,
hidden_dim=50,
max_iter=50,
eta=0.05)
 _ = torch_rnn.fit(X_train, y_train )<predict_on_test>
def sigmoid(x, L ,x0, k, b):
y = -L /(1 + np.exp(-k*(x-x0)))+b
return(y )
%%time
predictions = torch_rnn.predict(X_dev)
 print(classification_report(y_dev, predictions))<train_model>
print(( utility_noisy.mean() -utility_noisy.mean().max() ).sort_values().corr(( scores_noisy.mean() -scores_noisy.mean().max() ).sort_values()))