Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<compute_train_metric>
tf.random.set_seed(42)
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
embedding_matrix_glov = np.zeros(( num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max: df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max: df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max: df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max: df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max: df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max: df[col] = df[col].astype(np.float32)
 else: df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
return df
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
embedding_matrix_para = np.zeros(( num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<normalization>
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train = reduce_mem_usage(train)
 features = [c for c in train.columns if 'feature' in c]
print('Concatting matrixes...')
 matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 print(matrix.shape)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()<train_model>
train = train.astype({c: np.float32 for c in train.select_dtypes(include='float16' ).columns})
 train = train.fillna(train.mean())
 f_mean = np.mean(train[features[1:]].values,axis=0)
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train.weight != 0]
n_folds = 5
seed = 2020
skf = StratifiedKFold(n_splits=n_folds, shuffle=False)
 resp_cols = ['resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4']
X_train = train[train['date'] < 400][features]
X_test = train[train['date'] >= 400][features]
y_train = np.stack([(train[train['date'] < 400][c] > 0.000001 ).astype('int')for c in resp_cols] ).T
y_test = np.stack([(train[train['date'] >= 400][c] > 0.000001 ).astype('int')for c in resp_cols] ).T
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss<set_options>
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_train, dtype=torch.long ).cuda() ,
torch.tensor(y_train, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=True)
 val_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_val, dtype=torch.long ).cuda() ,
torch.tensor(y_val, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long ).cuda() ,
torch.tensor(y_test, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=False)
 class Sentiment(nn.Module):
def __init__(self,matrix,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',matrix.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding(matrix.shape[0],matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=matrix.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 _, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
out = self.linear1(F.relu(out))
return self.linear2(self.dropout(out))
model = Sentiment(matrix, batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,(x_batch,y_true)in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(x_batch ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,(x_val_batch,y_true)in enumerate(list(val_loader),1):
y_pred = model(x_val_batch ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )<load_pretrained>
TUNING_IND = False
def create_model(hp,input_dim,output_dim):
inputs = tf.keras.layers.Input(input_dim)
 x = tf.keras.layers.BatchNormalization()(inputs)
 x = tf.keras.layers.GaussianNoise(hp.Choice('noise',[0.0,0.03,0.05]))(x)
 x = tf.keras.layers.Dropout(hp.Choice('init_dropout',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(hp.Int('num_units_1', 128, 2048, 64), activation=hp.Choice('activation_1', ['tanh','relu','swish']))(x)
 x = tf.keras.layers.Dropout(hp.Choice(f'dropout_1',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(hp.Int('num_units_2', 128, 1024, 32), activation=hp.Choice('activation_2', ['tanh','relu','swish']))(x)
 x = tf.keras.layers.Dropout(hp.Choice(f'dropout_2',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 model.compile(optimizer=tf.keras.optimizers.Adam(hp.Choice('lr',[1e-2, 1e-3, 1e-5])) ,loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=hp.Choice('label_smoothing',[0.0, 0.01, 0.1])) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = tf.keras.Sequential([
tf.keras.Input(shape = len(features)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.GaussianNoise(0.05),
tf.keras.layers.Dropout(0.00),
tf.keras.layers.Dense(2048, activation='tanh'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(1024, activation='swish'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(2048, activation='swish'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(128, activation='swish'),
tf.keras.layers.Dense(5, activation = 'sigmoid')
 ])
 EPOCHS = 500
BATCH_SIZE = 1024
if TUNING_IND:
EPOCHS = 50
MAX_TRIAL = 20
model_fn = lambda hp: create_model(hp, X_train.shape[-1], y_train.shape[-1])
 tuner = kt.tuners.BayesianOptimization(model_fn, kt.Objective('val_auc', direction='max'), MAX_TRIAL, seed = 2020)
 tuner.search(X_train, y_train, epochs=EPOCHS, validation_data=(X_test, y_test),callbacks=[tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 10, restore_best_weights=True)])
 model = tuner.get_best_models() [0]
else:
callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True)
 optimizer = tf.keras.optimizers.SGD(learning_rate=1e-5)
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.00)
 model.compile(loss = loss, optimizer=optimizer, metrics=[tf.keras.metrics.AUC() ])
 history = model.fit(X, y, batch_size=BATCH_SIZE, epochs=EPOCHS, callbacks=[callback], validation_data=(X_test, y_test))
print(os.listdir())
 model = Sentiment(matrix, batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))<train_model>
print('Threshold:',search_result['threshold'])
 print(test_X.shape)
 submission_list = list(torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(test_X, dtype=torch.long ).cuda()),
batch_size=batch_size,
shuffle=False))
pred = []
with torch.no_grad() :
model.eval()
for(x,)in submission_list:
y_pred = torch.sigmoid(model(x ).squeeze(1)).detach()
pred += y_pred.cpu().numpy().tolist()
pred = np.array(pred)
 df_subm = pd.read_csv('.. /input/sample_submission.csv')
 df_subm.prediction =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv("submission.csv", index=False )<import_modules>
Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU,
Conv1D ,GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D,
Conv2D, MaxPool2D, concatenate,
Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, Bidirectional, 
)
 %matplotlib inline<define_variables>
DEBUG = False
EMBED_SIZE = 300
MAX_FEATURES = 10000 if DEBUG else 90000
SEQUENCE_LENGTH = 50
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
WIKI = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'<load_from_csv>
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
action = np.mean(model(x_tt, training = False ).numpy() [0])
 if(action > 0.502):
sample_prediction_df.action = 1
else:
sample_prediction_df.action = 0
else:
sample_prediction_df.action = 0
env.predict(sample_prediction_df )