Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<define_variables>
SEED = 1111
inference = False
cv = False
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train_pickle_file = '/kaggle/input/pickling/train.csv.pandas.pickle'
train = pickle.load(open(train_pickle_file, 'rb'))
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 train['bias'] = 1
features = [c for c in train.columns if "feature" in c]
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<feature_engineering>
def build_neutralizer(train, features, proportion, return_neut=False):

 neutralizer = {}
neutralized_features = np.zeros(( train.shape[0], len(features)))
 target = train[['resp', 'bias']].values
for i, f in enumerate(features):
feature = train[f].values.reshape(-1, 1)
 coeffs = np.linalg.lstsq(target, feature)[0]
neutralized_features[:, i] =(feature -(proportion * target.dot(coeffs)) ).squeeze()
neutralizer = np.linalg.lstsq(train[features+['bias']].values, neutralized_features)[0]
if return_neut:
return neutralized_features, neutralizer
else:
return neutralizer
def neutralize_array(array, neutralizer):
neutralized_array = array.dot(neutralizer)
 return neutralized_array
def test_neutralization() :
dummy_train = train.loc[:100000, :]
proportion = 1.0
neutralized_features, neutralizer = build_neutralizer(dummy_train, features, proportion, True)
 dummy_neut_train = neutralize_array(dummy_train[features+['bias']].values, neutralizer)
 print(neutralized_features[0, :10], dummy_neut_train[0, :10])
 test_neutralization()
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
x_train = train_df["question_text"].fillna("_
x_test = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 y_train = train_df['target'].values<statistical_test>
proportion = 1.0
neutralizer = build_neutralizer(train, features, proportion)
 train[features] = neutralize_array(train[features+['bias']].values, neutralizer )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train.loc[:, train.columns.str.contains('feature')]
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
seed_everything()
glove_embeddings = load_glove(tokenizer.word_index)
 paragram_embeddings = load_para(tokenizer.word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 np.shape(embedding_matrix )<split>
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
epochs = 200
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 if inference:
clf = keras.models.load_model('keras_nn')
 else:
clf.fit(X, y, epochs=epochs, batch_size=5000)
 clf.save('keras_nn')
 models = []
models.append(clf)
 th = 0.5000
f = np.median
models = models[-3:]
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=10 ).split(x_train, y_train))<normalization>
if cv:
oof_probas = np.zeros(y.shape)
 val_idx_all = []
N_SPLITS = 5
gkf = GroupKFold(n_splits=N_SPLITS)
 for fold,(train_idx, val_idx)in enumerate(gkf.split(train.action.values, groups=train.date.values)) :
X_train, X_val = X.iloc[train_idx], X.iloc[val_idx].values
y_train, y_val = y[train_idx], y[val_idx]
clf.fit(X_train, y_train, epochs=epochs, batch_size=5000)
 oof_probas[val_idx] += clf(X_val, training=False ).numpy()
score = roc_auc_score(y_val, oof_probas[val_idx])
 print(f'FOLD {fold} ROC AUC:\t {score}')
 del X_train, X_val, y_train, y_val
gc.collect()
val_idx_all.append(val_idx)
 val_idx = np.concatenate(val_idx_all)
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
if cv:
auc_oof = roc_auc_score(y[val_idx], oof_probas[val_idx])
 print(auc_oof )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(480, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<define_variables>
def determine_action(df, thresh):

 action =(df.weight * df.resp > thresh ).astype(int)
 return action
def date_weighted_resp(df):

 cols = ['weight', 'resp', 'action']
weighted_resp = np.prod(df[cols], axis=1)
 return weighted_resp.sum()
def calculate_t(dates_p):

 e_1 = dates_p.sum() / np.sqrt(( dates_p**2 ).sum())
 e_2 = np.sqrt(250/np.abs(len(dates_p)))
 return e_1 * e_2
def calculate_u(df, thresh):

 df = df.copy()
dates_p = df.groupby('date' ).apply(date_weighted_resp)
 t = calculate_t(dates_p)
 return t, min(max(t, 0), 6)* dates_p.sum()
def plot_roc_curve(fpr, tpr, label=None):
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, linewidth=2, label=label)
 plt.plot([0, 1], [0, 1], 'k--', label='Random')
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.legend(loc='lower right')
 plt.grid()
def plot_precision_recall_curve(precisions, recalls, thresholds):
plt.figure(figsize=(8, 6))
plt.plot(thresholds, precisions[:-1], 'b--', label='Precision')
 plt.plot(thresholds, recalls[:-1], 'g-', label='Recall')
 plt.xlabel('Thresholds')
 plt.legend(loc='lower left')
 plt.grid()
def plot_thresh_u_t(df, oof):
threshs = np.linspace(0, 1, 1000)
 ts = []
us = []
for thresh in threshs:
df['action'] = np.where(oof >= thresh, 1, 0)
 t, u = calculate_u(df, thresh)
 ts.append(t)
 us.append(u)
 ts = np.array(ts)
 us = np.array(us)
 ts = np.where(np.isnan(ts), 0.0, ts)
 us = np.where(np.isnan(us), 0.0, us)
 tmax = np.argmax(ts)
 umax = np.argmax(us)
 print(f'Max Utility Score: {us[umax]}')
 fig, axes = plt.subplots(1, 2, figsize=(14, 4))
axes[0].plot(threshs, ts)
 axes[0].set_title('Different t scores by threshold')
 axes[0].set_xlabel('Threshold')
 axes[0].axvline(threshs[tmax])
 axes[1].plot(threshs, us)
 axes[1].set_title('Different u scores by threshold')
 axes[1].set_xlabel('Threshold')
 axes[1].axvline(threshs[umax], color='r', linestyle='--', linewidth=1.2)
 print(f'Optimal Threshold: {threshs[umax]}')
 return threshs[umax]
<choose_model_class><EOS> 
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
x_tt = np.append(x_tt, [[1]], axis=1)
 x_tt = neutralize_array(x_tt, neutralizer)
 pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= opt_thresh, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )