Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<create_dataframe>
warnings.filterwarnings('ignore')
df = pd.DataFrame(gscv.cv_results_)
 df<choose_model_class>
train = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features[1:]].mean()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 print("Converting")
 train = train.to_pandas()
f_mean = f_mean.values.get()
np.save('f_mean.npy', f_mean)
%%time
clf = MultinomialNB()
pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<find_best_model_class>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing,learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.RectifiedAdam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model
%%time
clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<train_on_grid>
tf.random.set_seed(2020)
 batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.20, 0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
oof = np.zeros(len(train['action']))
def utility_gain(pred_val,test_index):
uniqueValues = train.loc[test_index,'date'].unique()
min_day = uniqueValues.min().astype('int32')
 max_day = uniqueValues.max().astype('int32')
 nro_days = max_day - min_day +1
print(min_day)
 print(max_day)
 pi = np.zeros(nro_days)
 for j in range(min_day, max_day):
index = train.index[train['date'] == j].intersection(test_index)
 m = train.loc[index, 'weight'] * train.loc[index,'resp'] * pred_val[index]
pi[j-min_day] = np.sum(m)
 t =(np.sum(pi)/ math.sqrt(np.sum(pi**2)))* math.sqrt(250/nro_days)
 utility = min(max(t,0),6)* np.sum(pi)
 return utility
for fold,(tr,te)in enumerate(GroupTimeSeriesSplit().split(train, groups=train['date'])) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
ckp_path = f'Model_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 5, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr,
validation_data =(X_val, y_val),
epochs = 200,
batch_size = batch_size,
callbacks = [rlr, ckp, es],
verbose = 0)
 oof[te] += model.predict(X_val, batch_size = batch_size * 4 ).ravel()
score = roc_auc_score(y_val, oof[te])
 print(f'Fold {fold} ROC AUC:\t', score)
 pred_val = model.predict(X_val ).ravel()
pred_val = pd.Series(pred_val, index=te)
 pred_val = pred_val.round().astype(int)
 utility = utility_gain(pred_val,te)
 print("%.2f" % utility)
 K.clear_session()
rubbish = gc.collect()
pipeline = Pipeline([('tfidf', TfidfVectorizer()),('clf', MultinomialNB())])
 parameters = [
{
'clf__alpha': [1, 0, 0.5],
'clf__fit_prior': [True, False]
},
]
os.write(1, b"Starting grid search of alpha & fit_prior
")
 gscv = GridSearchCV(pipeline, parameters, cv=3, n_jobs=12, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"] )<create_dataframe>
score_oof = roc_auc_score(train['action'].values, oof)
 print(score_oof )
df = pd.DataFrame(gscv.cv_results_)
 df<choose_model_class>
models = []
for i in range(2):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates,label_smoothing, learning_rate)
 clf.load_weights(f'./Model_{i}.hdf5')
 models.append(clf )
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<find_best_params>
f_mean = np.load('./f_mean.npy')
%%time
!pip install gensim
fastText_model = FastText.load('/kaggle/input/english-wikipedia-articles-20170820-models/enwiki_2017_08_20_fasttext.model')
 def randvec(w, n=50, lower=-1.0, upper=1.0):

 return np.array([random.uniform(lower, upper)for i in range(n)])
 def get_oov_fasttext(w):
twograms = ngrams(w, 2)
 vectors = []
for gram in twograms:
word_2gram = gram[0] + gram[1]
if word_2gram in fastText_model:
vectors.append(fastText_model[word_2gram])
 if(len(vectors)> 0):
return np.sum(vectors, axis=0)
 else:
return randvec(w, n=300)
 def fasttext_vec(w):

 if(w in fastText_model):
return fastText_model[w]
else:
return get_oov_fasttext(w)
 class FastTextTransformer(BaseEstimator, TransformerMixin):
def __init__(self, combine_strategy="concatenate", max_sentence_length=30):
assert(combine_strategy=="concatenate" or combine_strategy=='mean')
 self.combine_strategy = combine_strategy
self.max_sentence_length = max_sentence_length
self.empty_word_token = "EOF"
def transform(self, text_list):
texts = text_list.tolist()
result = [];
for text in texts:
vectors = [];
words = text.split()
if(self.combine_strategy == 'concatenate'):
max_index = self.max_sentence_length
else:
max_index = len(words)
 for index in range(max_index):
if(len(words)> index):
word = words[index]
else:
word = self.empty_word_token
vectors.append(fasttext_vec(word))
if(self.combine_strategy == 'concatenate'):
result.append(np.concatenate(vectors))
elif(self.combine_strategy == 'mean'):
result.append(np.mean(vectors, axis=0))
return result;
def fit(self, X, y=None):

 return self<train_on_grid>
env = janestreet.make_env()
env_iter = env.iter_test()
<choose_model_class><EOS> 
opt_th = 0.501
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
for i, clf in enumerate(models):
if i == 0:
pred = clf(x_tt, training = False ).numpy().item() / len(models)
 else:
pred += clf(x_tt, training = False ).numpy().item() / len(models)
 pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )