Kaggle competition: Jane Street Market Prediction
test['isup_grade'] = np.argmax(preds, axis=1)
 test.drop('data_provider', axis=1, inplace=True)
 test.to_csv('submission.csv', index=False)
 print('Submission saved' )<categorify>
pca_components = 60
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape )<compute_train_metric>
e_size = 64
fc_input = pca_components
h_dims = [512,512,256,128]
dropout_rate = 0.5
epochs = 200
minibatch_size = 100000
class MarketPredictor(nn.Module):
def __init__(self):
super(MarketPredictor, self ).__init__()
self.e = nn.Embedding(2,e_size)
 self.deep = nn.Sequential(
 nn.Linear(fc_input,h_dims[0]),
nn.BatchNorm1d(h_dims[0]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[0],h_dims[1]),
nn.BatchNorm1d(h_dims[1]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[1],h_dims[2]),
nn.BatchNorm1d(h_dims[2]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[2],h_dims[3]),
nn.BatchNorm1d(h_dims[3]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[3],e_size),
nn.BatchNorm1d(e_size),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate)
)
 self.reduce = nn.utils.weight_norm(nn.Linear(e_size,5))
def forward(self,xi,xf):
e_out = self.e(xi)
 f_out = self.deep(xf)
 ef_out = self.reduce(e_out+f_out)
 return ef_out
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
epochs = 200
path = '/kaggle/input/pytorch-nn-model-more-feature-engineering/marketpredictor_state_dict_'+str(epochs)+'epochs.pt'
model = MarketPredictor()
model.load_state_dict(torch.load(path,map_location=dev))
model.to(dev)
 model.eval()
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<feature_engineering>
with open('/kaggle/input/pytorch-nn-model-more-feature-engineering/feature_processing.pkl', 'rb')as f:
sc, pca, maxindex, fill_val, remove_names= pickle.load(f )
matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()<load_from_csv>
feature_names = ['feature_'+str(i)for i in range(1,130)]
exclude = np.where([maxindex[i,1] > 100 and maxindex [i,2] > 1 for i in range(129)])[0]
keep = np.where([(feature_names[i] != remove_names ).all() for i in range(129)])[0]
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )<import_modules>
env = janestreet.make_env()
iter_test = env.iter_test()
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
def get_model(adam, trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 filters = []
for f in [3,4,5]:
conv = Conv1D(64,f, activation='elu' )(emb)
 pool = GlobalMaxPooling1D()(conv)
 filters.append(pool)
 x = concatenate(filters)
 x = Dense(max_len, activation="relu" )(x)
 x = Dropout(0.5 )(x)
 x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('Pretraining finished, unfreezing embeddings layer...')
 model = get_model(adam,True)
 model.load_weights(model_file)
 modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
all_results[model_file] = search_result['f1']
if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
 print('-'*80 )<predict_on_test>
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() == 0:
sample_prediction_df.action = 0
else:
test_df_features = test_df[feature_names].to_numpy()
for i in exclude:
if test_df_features[0,i] == maxindex[i,0]:
test_df_features[0,i] = fill_val[i]
test_df_int_features = test_df['feature_0'].to_numpy()
nans = np.isnan(test_df_features)
 for i in range(129):
if nans[0,i]:
test_df_features[0,i] = fill_val[i]
test_df_features = test_df_features[:,keep]
test_df_features_scaled = sc.transform(test_df_features)
 test_df_features_pca=pca.transform(test_df_features_scaled)
 itensor = torch.tensor(( test_df_int_features+1)//2,dtype=torch.long,device=dev)
 ftensor = torch.tensor(test_df_features_pca,dtype=torch.float,device=dev)
 s = model(itensor,ftensor)[0,0].item()
sample_prediction_df.action = int(np.round(1/(1+np.exp(-s))))
env.predict(sample_prediction_df)