Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<feature_engineering>
import numpy as np
import pandas as pd
from tensorflow.keras.callbacks import TensorBoard
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
matrixes = [embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()<import_modules>
from tensorflow import keras
from tensorflow.keras.layers import MaxPooling1D, Dense, LeakyReLU, Conv1D
from tensorflow.keras.layers import Flatten, Activation, BatchNormalization, Dropout
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras import layers
from kerastuner.tuners import RandomSearch
from kerastuner.engine.hyperparameters import HyperParameters
import time
import pickle
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
batch_size = int(x_train.shape[0]*K.eval(adam.lr))
print('Batch size = ',batch_size)
 inp1 = Input(shape=(max_len,))
x = Embedding(num_words, dim, weights=[matrix],trainable = False, )(inp1)
 x = SpatialDropout1D(0.1 )(x)
 x1 = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x1)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)
 x = Add()([x1,x])
 atten_1 = Attention(max_len )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([atten_1, avg_pool, max_pool])
 x = Dense(100, activation="relu",kernel_regularizer=regularizers.l2(0.0001))(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy',f1])
 print(model.summary())
 patience = 5
modelcheck = ModelCheckpoint('model.h5',save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = ReduceLROnPlateau(patience=1,verbose=2)
 history = model.fit(x_train,y_train,
batch_size=batch_size,
validation_split=0.2,
epochs=100,
callbacks=[modelcheck,stop,reduce])
 print('training finished...' )<predict_on_test>
import tensorflow as tf
model.load_weights('model.h5')
 y_pred = model.predict(x_train,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))<save_to_csv>
%%time
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True )
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<import_modules>
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train.loc[:, train.columns.str.contains('feature')]
Y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
leaky_relu_alpha =0.05
LeakyReLU(alpha=leaky_relu_alpha)
import numpy as np
import pandas as pd
import datetime as datetime
from sklearn import preprocessing<load_pretrained>
def build_model() :
model = keras.models.Sequential()
model.add(Conv1D(180, 2, input_shape=x_train.shape[1:]))
model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=leaky_relu_alpha))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.15))
model.add(Flatten())
 model.add(Dense(180))
model.add(LeakyReLU(alpha=leaky_relu_alpha))
model.add(Dropout(0.15))
model.add(Dense(5))
model.add(Activation("sigmoid"))
model.compile(optimizer=keras.optimizers.Adam(lr=1e-3),
loss=BinaryCrossentropy(label_smoothing=0.095),
metrics=[tf.keras.metrics.AUC(name = 'auc'), "accuracy"])
 return model
register_matplotlib_converters() 
<import_modules>
model = build_model()
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<load_from_csv>
model.fit(x=x_train,
y=Y,
epochs=10,
batch_size=1024 )
training_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 training_set.shape<load_from_csv>
from tqdm import tqdm
<define_variables><EOS> 
f = np.median
th = 0.5000
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt.reshape(-1, x_tt.shape[1], 1), training = False ).numpy() ],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )